diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb47ed28..94cc4e61 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,10 +20,10 @@ jobs: with: distribution: "temurin" java-version: ${{ env.JAVA_VERSION }} - + - name: Checkout uses: actions/checkout@v4 - + - name: Install Go uses: actions/setup-go@v5 with: @@ -32,11 +32,11 @@ jobs: - name: Install project tools and dependencies shell: bash run: make project-tools - + - name: Download OAS shell: bash run: make download-oas - + - name: Generate SDK shell: bash run: make generate-go-sdk @@ -54,15 +54,20 @@ jobs: with: distribution: "temurin" java-version: ${{ env.JAVA_VERSION }} - + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION_BUILD }} + - name: Checkout uses: actions/checkout@v4 - + - name: install uv uses: astral-sh/setup-uv@v7 with: version: "0.10.4" - + - name: Install project tools and dependencies shell: bash run: | @@ -70,29 +75,29 @@ jobs: . .venv/bin/activate python -m pip install --upgrade pip make project-tools LANGUAGE=python - + - name: Download OAS shell: bash run: make download-oas - + - name: Generate SDK shell: bash run: make generate-python-sdk - + - name: install uv uses: astral-sh/setup-uv@v7 with: version: "0.10.4" python-version: ${{ matrix.python-version }} - + - name: Install SDK working-directory: ./sdk-repo-updated run: make install-dev - + - name: Lint working-directory: ./sdk-repo-updated run: make lint - + - name: Test working-directory: ./sdk-repo-updated run: make test @@ -109,20 +114,25 @@ jobs: with: distribution: "temurin" java-version: ${{ matrix.java-version }} - + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION_BUILD }} + - name: Checkout uses: actions/checkout@v5 - + - name: Download OAS run: make download-oas - + - name: Generate SDK run: make generate-java-sdk - + - name: Lint working-directory: ./sdk-repo-updated run: make lint - + - name: Test working-directory: ./sdk-repo-updated run: make test diff --git a/.github/workflows/sdk-pr.yaml b/.github/workflows/sdk-pr.yaml index 4819c2ea..9224a88c 100644 --- a/.github/workflows/sdk-pr.yaml +++ b/.github/workflows/sdk-pr.yaml @@ -14,7 +14,7 @@ jobs: main-go: name: "[Go] Update SDK Repo" runs-on: ubuntu-latest - permissions: + permissions: pull-requests: write contents: write steps: @@ -23,44 +23,44 @@ jobs: with: key: ${{ secrets.SSH_PRIVATE_KEY }} known_hosts: ${{ vars.SSH_KNOWN_HOSTS }} - + - name: Install Java uses: actions/setup-java@v4 with: distribution: "temurin" java-version: ${{ env.JAVA_VERSION }} - + - name: Checkout uses: actions/checkout@v4 - + - name: Install Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - + - name: Install project tools and dependencies shell: bash run: make project-tools - + - name: Download OAS shell: bash run: make download-oas - + - name: Generate SDK shell: bash run: make generate-sdk - + - name: Push SDK env: GH_REPO: "stackitcloud/stackit-sdk-go" GH_TOKEN: ${{ secrets.SDK_PR_TOKEN }} run: | scripts/sdk-create-pr.sh "generator-bot-${{ github.run_id }}" "Generated from GitHub run [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" - + main-python: name: "[Python] Update SDK Repo" runs-on: ubuntu-latest - permissions: + permissions: pull-requests: write contents: write steps: @@ -69,16 +69,21 @@ jobs: with: key: ${{ secrets.SSH_PRIVATE_KEY }} known_hosts: ${{ vars.SSH_KNOWN_HOSTS }} - + - name: Install Java uses: actions/setup-java@v4 with: distribution: "temurin" java-version: ${{ env.JAVA_VERSION }} - + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Checkout uses: actions/checkout@v4 - + - name: Install project tools and dependencies shell: bash run: | @@ -86,20 +91,20 @@ jobs: . .venv/bin/activate python -m pip install --upgrade pip make project-tools LANGUAGE=python - + - name: install uv uses: astral-sh/setup-uv@v7 with: version: "0.10.4" - + - name: Download OAS shell: bash run: make download-oas - + - name: Generate SDK shell: bash run: make generate-sdk LANGUAGE=python - + - name: Push SDK env: GH_REPO: "stackitcloud/stackit-sdk-python" @@ -111,7 +116,7 @@ jobs: main-java: name: "[Java] Update SDK Repo" runs-on: ubuntu-latest - permissions: + permissions: pull-requests: write contents: write steps: @@ -120,24 +125,29 @@ jobs: with: key: ${{ secrets.SSH_PRIVATE_KEY }} known_hosts: ${{ vars.SSH_KNOWN_HOSTS }} - + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Install Java uses: actions/setup-java@v4 with: distribution: "temurin" java-version: ${{ env.JAVA_VERSION }} - + - name: Checkout uses: actions/checkout@v5 - + - name: Download OAS run: make download-oas - + - name: Generate SDK run: make generate-sdk env: LANGUAGE: java - + - name: Push SDK env: GH_REPO: "stackitcloud/stackit-sdk-java" diff --git a/scripts/build/delete.go b/scripts/build/delete.go new file mode 100644 index 00000000..cd749694 --- /dev/null +++ b/scripts/build/delete.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "flag" + "fmt" + "strings" +) + +func runDelete(ctx context.Context, args []string, io IO) bool { + deleteFlags := flag.NewFlagSet("delete", flag.ContinueOnError) + deleteFlags.SetOutput(io.Err) + + var planPath string + deleteFlags.StringVar(&planPath, "plan", "", "path to the generation plan") + if err := deleteFlags.Parse(args); err != nil { + return false + } + if len(deleteFlags.Args()) != 0 { + fmt.Fprintf(io.Err, "unexpected arguments: %s\n", strings.Join(deleteFlags.Args(), " ")) + return false + } + if planPath == "" { + fmt.Fprintln(io.Err, "--plan is required") + return false + } + + plan, err := readPlan(io.FS, planPath) + if err != nil { + fmt.Fprintf(io.Err, "read generation plan: %v\n", err) + return false + } + + for _, service := range plan.Services { + if service.Action == "delete" { + fmt.Fprintln(io.Out, service.Service) + } + } + return true +} diff --git a/scripts/build/delete_test.go b/scripts/build/delete_test.go new file mode 100644 index 00000000..8e012acf --- /dev/null +++ b/scripts/build/delete_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "bytes" + "context" + "testing" + "testing/fstest" +) + +func TestRunDeletePrintsServicesMarkedForDeletion(t *testing.T) { + filesystem := fstest.MapFS{ + "generation-plan.json": {Data: []byte(`{ + "services": [ + {"service": "blocked", "oas_service": "blocked", "action": "block"}, + {"service": "foo", "oas_service": "foo-bar", "action": "generate"}, + {"service": "obsolete", "action": "delete"}, + {"service": "service2", "action": "delete"} + ] +}`)}, + } + var stdout, stderr bytes.Buffer + + ok := runDelete(context.Background(), []string{"--plan", "generation-plan.json"}, IO{ + Out: &stdout, + Err: &stderr, + FS: readOnlyFileSystem{FileSystem: filesystem}, + }) + if !ok { + t.Fatalf("runDelete returned false: %s", stderr.String()) + } + if got, want := stdout.String(), "obsolete\nservice2\n"; got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} diff --git a/scripts/build/generate.go b/scripts/build/generate.go new file mode 100644 index 00000000..4d27d74d --- /dev/null +++ b/scripts/build/generate.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/fs" + "strings" +) + +func runGenerate(ctx context.Context, args []string, io IO) bool { + generateFlags := flag.NewFlagSet("generate", flag.ContinueOnError) + generateFlags.SetOutput(io.Err) + + var planPath string + generateFlags.StringVar(&planPath, "plan", "", "path to the generation plan") + if err := generateFlags.Parse(args); err != nil { + return false + } + if len(generateFlags.Args()) != 0 { + fmt.Fprintf(io.Err, "unexpected arguments: %s\n", strings.Join(generateFlags.Args(), " ")) + return false + } + if planPath == "" { + fmt.Fprintln(io.Err, "--plan is required") + return false + } + + plan, err := readPlan(io.FS, planPath) + if err != nil { + fmt.Fprintf(io.Err, "read generation plan: %v\n", err) + return false + } + + for _, service := range plan.Services { + if service.Action == "generate" { + // use null byte as field separator to avoid parsing/escaping issues + fmt.Fprintf(io.Out, "%s\x00%s\n", service.OASService, service.Service) + } + } + return true +} + +func readPlan(filesystem fs.FS, path string) (Plan, error) { + contents, err := fs.ReadFile(filesystem, path) + if err != nil { + return Plan{}, err + } + + var plan Plan + if err := json.Unmarshal(contents, &plan); err != nil { + return Plan{}, err + } + return plan, nil +} diff --git a/scripts/build/generate_test.go b/scripts/build/generate_test.go new file mode 100644 index 00000000..18334424 --- /dev/null +++ b/scripts/build/generate_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "bytes" + "context" + "io/fs" + "testing" + "testing/fstest" +) + +func TestRunGeneratePrintsServicesMarkedForGeneration(t *testing.T) { + filesystem := fstest.MapFS{ + "generation-plan.json": {Data: []byte(`{ + "services": [ + {"service": "blocked", "oas_service": "blocked", "action": "block"}, + {"service": "foo", "oas_service": "foo-bar", "action": "generate"}, + {"service": "obsolete", "action": "delete"}, + {"service": "service2", "oas_service": "service-2", "action": "generate"} + ] +}`)}, + } + var stdout, stderr bytes.Buffer + + ok := runGenerate(context.Background(), []string{"--plan", "generation-plan.json"}, IO{ + Out: &stdout, + Err: &stderr, + FS: readOnlyFileSystem{FileSystem: filesystem}, + }) + if !ok { + t.Fatalf("runGenerate returned false: %s", stderr.String()) + } + if got, want := stdout.String(), "foo-bar\x00foo\nservice-2\x00service2\n"; got != want { + t.Errorf("stdout = %q, want %q", got, want) + } +} + +type readOnlyFileSystem struct { + FileSystem fstest.MapFS +} + +func (filesystem readOnlyFileSystem) Open(name string) (fs.File, error) { + return filesystem.FileSystem.Open(name) +} + +func (filesystem readOnlyFileSystem) WriteFile(string, []byte, fs.FileMode) error { + return nil +} diff --git a/scripts/build/go.mod b/scripts/build/go.mod new file mode 100644 index 00000000..92f03332 --- /dev/null +++ b/scripts/build/go.mod @@ -0,0 +1,3 @@ +module github.com/stackitcloud/stackit-sdk-generator/scripts/build + +go 1.24 diff --git a/scripts/build/languages.go b/scripts/build/languages.go new file mode 100644 index 00000000..1da56e22 --- /dev/null +++ b/scripts/build/languages.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "strings" +) + +type Language interface { + NormalizeServiceName(string) string +} + +type goLanguage struct{} + +func (goLanguage) NormalizeServiceName(service string) string { + return normalizeAlphaNumeric(service) +} + +type pythonLanguage struct{} + +func (pythonLanguage) NormalizeServiceName(service string) string { + return normalizeAlphaNumeric(service) +} + +type javaLanguage struct{} + +func (javaLanguage) NormalizeServiceName(service string) string { + name := normalizeAlphaNumeric(service) + if name != "" && name[0] >= '0' && name[0] <= '9' { + return "_" + name + } + return name +} + +func newLanguage(name string) (Language, error) { + switch name { + case "go": + return goLanguage{}, nil + case "python": + return pythonLanguage{}, nil + case "java": + return javaLanguage{}, nil + default: + return nil, fmt.Errorf("unsupported language %q", name) + } +} + +func normalizeAlphaNumeric(service string) string { + var normalized strings.Builder + for _, char := range strings.ToLower(service) { + if (char >= 'a' && char <= 'z') || (char >= '0' && char <= '9') { + normalized.WriteRune(char) + } + } + return normalized.String() +} diff --git a/scripts/build/languages_test.go b/scripts/build/languages_test.go new file mode 100644 index 00000000..afa190d5 --- /dev/null +++ b/scripts/build/languages_test.go @@ -0,0 +1,28 @@ +package main + +import "testing" + +func TestLanguageNormalizeServiceName(t *testing.T) { + tests := []struct { + language string + service string + want string + }{ + {language: "go", service: "Foo-_ Bar!", want: "foobar"}, + {language: "python", service: "Foo-_ Bar!", want: "foobar"}, + {language: "java", service: "Foo-_ Bar!", want: "foobar"}, + {language: "java", service: "123-service", want: "_123service"}, + } + + for _, test := range tests { + t.Run(test.language+"/"+test.service, func(t *testing.T) { + language, err := newLanguage(test.language) + if err != nil { + t.Fatal(err) + } + if got := language.NormalizeServiceName(test.service); got != test.want { + t.Errorf("NormalizeServiceName(%q) = %q, want %q", test.service, got, test.want) + } + }) + } +} diff --git a/scripts/build/main.go b/scripts/build/main.go new file mode 100644 index 00000000..4cf8dc79 --- /dev/null +++ b/scripts/build/main.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "io/fs" + "os" + "os/signal" + "syscall" +) + +func main() { + ctx := context.Background() + ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) + defer stop() + + workingDirectory, err := os.Getwd() + if err != nil { + panic(err) + } + root, err := os.OpenRoot(workingDirectory) + if err != nil { + panic(err) + } + defer root.Close() + + stdio := IO{ + Args: os.Args[1:], + In: os.Stdin, + Out: os.Stdout, + Err: os.Stderr, + FS: rootFileSystem{root: root}, + } + + if ok := run(ctx, stdio); !ok { + os.Exit(1) + } +} + +type FileSystem interface { + fs.FS + WriteFile(name string, data []byte, perm fs.FileMode) error +} + +type rootFileSystem struct { + root *os.Root +} + +func (filesystem rootFileSystem) Open(name string) (fs.File, error) { + return filesystem.root.Open(name) +} + +func (filesystem rootFileSystem) WriteFile(name string, data []byte, perm fs.FileMode) error { + file, err := filesystem.root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + written, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if written != len(data) { + return io.ErrShortWrite + } + return closeErr +} + +type IO struct { + Args []string + In io.Reader + Out io.Writer + Err io.Writer + FS FileSystem +} + +type Global struct { + Verbose bool + Language string +} + +func run(ctx context.Context, io IO) bool { + var global Global + globalF := flag.NewFlagSet("build", flag.ContinueOnError) + globalF.SetOutput(io.Err) + globalF.BoolVar(&global.Verbose, "verbose", false, "enable verbose output") + globalF.StringVar(&global.Language, "language", "go", "the programming language to use (go, python, java)") + if err := globalF.Parse(io.Args); err != nil { + return false + } + + args := globalF.Args() + if len(args) == 0 { + fmt.Fprintln(io.Err, "expected a command") + return false + } + + language, err := newLanguage(global.Language) + if err != nil { + fmt.Fprintln(io.Err, err) + return false + } + + switch args[0] { + case "plan": + return runPlan(ctx, global, language, args[1:], io) + case "generate": + return runGenerate(ctx, args[1:], io) + case "delete": + return runDelete(ctx, args[1:], io) + default: + fmt.Fprintf(io.Err, "unknown command %q\n", args[0]) + return false + } +} diff --git a/scripts/build/plan.go b/scripts/build/plan.go new file mode 100644 index 00000000..a09bf896 --- /dev/null +++ b/scripts/build/plan.go @@ -0,0 +1,187 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/fs" + "sort" + "strings" +) + +type PlanParams struct { + SpecDir string + ServiceDir string + BlocklistPath string + OutputPath string + IncludedService map[string]struct{} +} + +type Plan struct { + Services []PlannedService `json:"services"` +} + +type PlannedService struct { + Service string `json:"service"` + OASService string `json:"oas_service,omitempty"` + Action string `json:"action"` +} + +func runPlan(ctx context.Context, global Global, language Language, args []string, io IO) bool { + planFlags := flag.NewFlagSet("plan", flag.ContinueOnError) + planFlags.SetOutput(io.Err) + + var params PlanParams + var includedServices []string + planFlags.StringVar(¶ms.SpecDir, "spec-dir", "", "directory containing OAS service directories") + planFlags.StringVar(¶ms.ServiceDir, "service-dir", "", "directory containing generated service directories") + planFlags.StringVar(¶ms.BlocklistPath, "blocklist", "", "path to the language blocklist") + planFlags.StringVar(¶ms.OutputPath, "output", "", "path to write the JSON plan") + planFlags.Func("include-service", "service eligible for generation (may be repeated)", func(service string) error { + includedServices = append(includedServices, service) + return nil + }) + if err := planFlags.Parse(args); err != nil { + return false + } + if len(planFlags.Args()) != 0 { + fmt.Fprintf(io.Err, "unexpected arguments: %s\n", strings.Join(planFlags.Args(), " ")) + return false + } + params.IncludedService = make(map[string]struct{}, len(includedServices)) + for _, service := range includedServices { + params.IncludedService[service] = struct{}{} + } + + if err := validatePlanParams(params); err != nil { + fmt.Fprintf(io.Err, "invalid plan arguments: %v\n", err) + return false + } + + plan, err := createPlan(io.FS, params, language) + if err != nil { + fmt.Fprintf(io.Err, "create plan: %v\n", err) + return false + } + var formatted []byte + if formatted, err = formatPlan(plan); err != nil { + fmt.Fprintf(io.Err, "format plan: %v\n", err) + return false + } + if _, err = io.Out.Write(formatted); err != nil { + fmt.Fprintf(io.Err, "print plan: %v\n", err) + return false + } + if err = io.FS.WriteFile(params.OutputPath, formatted, 0o644); err != nil { + fmt.Fprintf(io.Err, "write plan: %v\n", err) + return false + } + + if global.Verbose { + fmt.Fprintf(io.Err, "wrote plan for %d services to %s\n", len(plan.Services), params.OutputPath) + } + return true +} + +func validatePlanParams(params PlanParams) error { + for name, value := range map[string]string{ + "spec-dir": params.SpecDir, + "service-dir": params.ServiceDir, + "blocklist": params.BlocklistPath, + "output": params.OutputPath, + } { + if value == "" { + return fmt.Errorf("--%s is required", name) + } + } + return nil +} + +func createPlan(filesystem FileSystem, params PlanParams, language Language) (Plan, error) { + blocked, err := readBlocklist(filesystem, params.BlocklistPath) + if err != nil { + return Plan{}, err + } + + specEntries, err := fs.ReadDir(filesystem, params.SpecDir) + if err != nil { + return Plan{}, fmt.Errorf("read spec directory: %w", err) + } + services := make(map[string]PlannedService, len(specEntries)) + for _, entry := range specEntries { + if !entry.IsDir() { + continue + } + + oasService := entry.Name() + service := language.NormalizeServiceName(oasService) + if service == "" { + return Plan{}, fmt.Errorf("OAS service %q has no valid service name", oasService) + } + if previous, exists := services[service]; exists { + return Plan{}, fmt.Errorf("OAS services %q and %q both normalize to %q", previous.OASService, oasService, service) + } + + action := "generate" + if _, isBlocked := blocked[service]; isBlocked { + action = "block" + } else if len(params.IncludedService) > 0 && !isIncluded(params.IncludedService, service) { + action = "not_included" + } + services[service] = PlannedService{Service: service, OASService: oasService, Action: action} + } + + serviceEntries, err := fs.ReadDir(filesystem, params.ServiceDir) + if err != nil { + return Plan{}, fmt.Errorf("read service directory: %w", err) + } + for _, entry := range serviceEntries { + if !entry.IsDir() { + continue + } + service := entry.Name() + if _, exists := services[service]; !exists { + services[service] = PlannedService{Service: service, Action: "delete"} + } + } + + plan := Plan{Services: make([]PlannedService, 0, len(services))} + for _, service := range services { + plan.Services = append(plan.Services, service) + } + sort.Slice(plan.Services, func(i, j int) bool { + return plan.Services[i].Service < plan.Services[j].Service + }) + return plan, nil +} + +func readBlocklist(filesystem FileSystem, path string) (map[string]struct{}, error) { + contents, err := fs.ReadFile(filesystem, path) + if err != nil { + return nil, err + } + + blocked := make(map[string]struct{}) + for line := range strings.SplitSeq(string(contents), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + blocked[line] = struct{}{} + } + return blocked, nil +} + +func isIncluded(included map[string]struct{}, service string) bool { + _, ok := included[service] + return ok +} + +func formatPlan(plan Plan) ([]byte, error) { + contents, err := json.MarshalIndent(plan, "", " ") + if err != nil { + return nil, err + } + return append(contents, '\n'), nil +} diff --git a/scripts/build/plan_test.go b/scripts/build/plan_test.go new file mode 100644 index 00000000..a40c572a --- /dev/null +++ b/scripts/build/plan_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io/fs" + "reflect" + "testing" + "testing/fstest" +) + +type mapFileSystem struct { + files fstest.MapFS + writes map[string][]byte +} + +func (filesystem *mapFileSystem) Open(name string) (fs.File, error) { + return filesystem.files.Open(name) +} + +func (filesystem *mapFileSystem) WriteFile(name string, data []byte, _ fs.FileMode) error { + filesystem.writes[name] = append([]byte(nil), data...) + return nil +} + +func TestRunPlan(t *testing.T) { + const outputPath = "generation-plan.json" + filesystem := &mapFileSystem{files: fstest.MapFS{ + "oas/services/blocked/v1/blocked.json": {Data: []byte{}}, + "oas/services/foo-bar/v1/foo-bar.json": {Data: []byte{}}, + "services/foobar/.keep": {Data: []byte{}}, + "services/obsolete/.keep": {Data: []byte{}}, + "languages/go/blocklist.txt": {Data: []byte("# blocked services\nblocked\n")}, + }, writes: make(map[string][]byte)} + var stdout, stderr bytes.Buffer + + ok := runPlan(context.Background(), Global{Language: "go"}, goLanguage{}, []string{ + "--spec-dir", "oas/services", + "--service-dir", "services", + "--blocklist", "languages/go/blocklist.txt", + "--output", outputPath, + }, IO{Out: &stdout, Err: &stderr, FS: filesystem}) + if !ok { + t.Fatalf("runPlan returned false: %s", stderr.String()) + } + + contents, ok := filesystem.writes[outputPath] + if !ok { + t.Fatalf("plan was not written to %q", outputPath) + } + var plan Plan + if err := json.Unmarshal(contents, &plan); err != nil { + t.Fatal(err) + } + if got, want := stdout.String(), string(contents); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } + want := []PlannedService{ + {Service: "blocked", OASService: "blocked", Action: "block"}, + {Service: "foobar", OASService: "foo-bar", Action: "generate"}, + {Service: "obsolete", Action: "delete"}, + } + if !reflect.DeepEqual(plan.Services, want) { + t.Errorf("services = %#v, want %#v", plan.Services, want) + } +} + +func TestRunPlanIncludeService(t *testing.T) { + const outputPath = "generation-plan.json" + filesystem := &mapFileSystem{ + files: fstest.MapFS{ + "oas/services/alice/v1/alice.json": {Data: []byte{}}, + "oas/services/bob/v1/bob.json": {Data: []byte{}}, + "oas/services/charlie/v1/charlie.json": {Data: []byte{}}, + "services/.keep": {Data: []byte{}}, + "languages/go/blocklist.txt": {Data: []byte(` + bob`, + )}, + }, + writes: make(map[string][]byte), + } + var stdout, stderr bytes.Buffer + + ok := runPlan(context.Background(), Global{Language: "go"}, goLanguage{}, []string{ + "--spec-dir", "oas/services", + "--service-dir", "services", + "--blocklist", "languages/go/blocklist.txt", + "--output", outputPath, + "--include-service", "alice", + "--include-service", "bob", + }, IO{Out: &stdout, Err: &stderr, FS: filesystem}) + if !ok { + t.Fatalf("runPlan returned false: %s", stderr.String()) + } + + contents, ok := filesystem.writes[outputPath] + if !ok { + t.Fatalf("plan was not written to %q", outputPath) + } + var plan Plan + if err := json.Unmarshal(contents, &plan); err != nil { + t.Fatal(err) + } + if got, want := stdout.String(), string(contents); got != want { + t.Errorf("stdout = %q, want %q", got, want) + } + want := []PlannedService{ + {Service: "alice", OASService: "alice", Action: "generate"}, + {Service: "bob", OASService: "bob", Action: "block"}, + {Service: "charlie", OASService: "charlie", Action: "not_included"}, + } + if !reflect.DeepEqual(plan.Services, want) { + t.Errorf("services = %#v, want %#v", plan.Services, want) + } +} diff --git a/scripts/build/readme.md b/scripts/build/readme.md new file mode 100644 index 00000000..16246d52 --- /dev/null +++ b/scripts/build/readme.md @@ -0,0 +1,51 @@ +# Stackit SDK Generator - Build Tool + +Currently we have a lot of bash scripts to build the SDKs. +In the long term we want to reduce this because of readbility issues. + +New build functionality should be implemented in this tool. +And if we have some time we can also refctor the existing bash scripts to use this tool. + +The end goal for this tool is to have a single entry point for building the SDKs. +Especially we want to leverage parallelism to speed up the build process. + +## Status Quo + +The build currently consists of multiple phases, split into multiple scripts: + +- download oas +- generate sdk +- push pr + +Information flows between these phases via files on disk. + +This design has the advantage that during local devlopment on the build system itself, you can easily run a single phase without having to run the entire build process. + +## Vision + +This build tool should also support multiple phases. +The dependencies between phases should be modeled explicitly. +So that a call like `build push-pr` will automatically trigger all required phases before the `push-pr` phase. + +It should be possible to skip phases that are up to date. +So if we run `build generate` the first time it would download OAS and generate the SDKs. +The second time it would only generate the SDKs, because the OAS files are already downloaded and up to date. + +### Parallelism + +The build tool should leverage parallelism to speed up the build process. +Currently most of the time is spent in the `generate` phase. +We serially call the SDK generator once for each service (including JVM startup). + +A good first optimization would be to call the SDK generator once with the batch command: +https://openapi-generator.tech/docs/usage#batch + +## Implemented Functionality + +Currently only 3 standalone commands are implemented: + +- plan.go: write a plan file, which services will be generated, which are blockesd, which should be deleted +- generate.go: helper command to get oas-service-name, on-disk-service-name pairs from the plan file to generate +- delete.go: helper command to get on-disk-service-name from the plan file to delete + +Language specific things like normalizing service names are implemented in languages.go diff --git a/scripts/generate-sdk/generate-sdk.sh b/scripts/generate-sdk/generate-sdk.sh index 68b12eea..8c0adefd 100755 --- a/scripts/generate-sdk/generate-sdk.sh +++ b/scripts/generate-sdk/generate-sdk.sh @@ -46,6 +46,18 @@ else exit 1 fi +# The generation plan is produced by scripts/build before SDK generation starts. +if type -p go >/dev/null; then + : +else + echo "Go not installed, unable to create the generation plan." + exit 1 +fi + +# build build tool +mkdir -p "${GENERATOR_PATH}" +go build -C "${ROOT_DIR}/scripts/build" -o "${GENERATOR_PATH}/build" + if [ ! -d ${ROOT_DIR}/oas ]; then echo "\"oas\" folder not found in root. Please add it manually or run \"make download-oas\"." exit 1 diff --git a/scripts/generate-sdk/languages/go.sh b/scripts/generate-sdk/languages/go.sh index 9d465e72..75216164 100644 --- a/scripts/generate-sdk/languages/go.sh +++ b/scripts/generate-sdk/languages/go.sh @@ -6,8 +6,6 @@ set -eo pipefail ROOT_DIR=$(git rev-parse --show-toplevel) SDK_REPO_LOCAL_PATH="${ROOT_DIR}/sdk-repo-updated" -OAS_REPO=https://github.com/stackitcloud/stackit-api-specifications - SDK_GO_VERSION="1.21" SERVICES_FOLDER="${SDK_REPO_LOCAL_PATH}/services" @@ -77,6 +75,15 @@ generate_go_sdk() { git checkout "${SDK_BRANCH}" make project-tools + ( + cd "${ROOT_DIR}" + "${ROOT_DIR}/scripts/bin/build" --language go plan \ + --spec-dir "oas/services" \ + --service-dir "sdk-repo-updated/services" \ + --blocklist "languages/golang/blocklist.txt" \ + --output "generation-plan.json" + ) + # Backup of the current state of the SDK services dir (services/) sdk_services_backup_dir=$(mktemp -d) if [[ ! ${sdk_services_backup_dir} || -d {sdk_services_backup_dir} ]]; then @@ -103,7 +110,7 @@ generate_go_sdk() { cd "${SDK_REPO_LOCAL_PATH}/core" go work use . fi - + # see https://openapi-generator.tech/docs/file-post-processing/ export GO_POST_PROCESS_FILE="gofmt -w" @@ -114,220 +121,119 @@ generate_go_sdk() { warning="" - for service_dir in "${ROOT_DIR}/oas/services"/*; do - service="${service_dir##*/}" - service="${service%.json}" + while IFS= read -r -d '' oas_service && IFS= read -r service; do + generate_go_service "${ROOT_DIR}/oas/services/${oas_service}" "${service}" "${oas_service}" + done < <( + "${ROOT_DIR}/scripts/bin/build" generate --plan "generation-plan.json" + ) - compat_layer_service_oas_name="${service}" + # Add examples to workspace + if [ -d "${EXAMPLES_FOLDER}" ]; then + for example_dir in ${EXAMPLES_FOLDER}/*; do + cd "${example_dir}" + go work use . + done + fi - # Remove invalid characters to ensure a valid Go pkg name - service="${service//-/}" # remove dashes - service="${service// /}" # remove empty spaces - service="${service//_/}" # remove underscores - service=$(echo "${service}" | tr '[:upper:]' '[:lower:]') # convert upper case letters to lower case - service=$(echo "${service}" | tr -d -c '[:alnum:]') # remove non-alphanumeric characters + # Add scripts to workspace + if [ -d "${SCRIPTS_FOLDER}" ]; then + cd "${SCRIPTS_FOLDER}" + go work use . + fi - go_pkg_name_format="^[a-z0-9]+$" - if [[ ! ${service} =~ ${go_pkg_name_format} ]]; then # check that it is a single lower case word - echo "Service ${service} has an invalid Go package name even after removing invalid characters. The generate-sdk.sh script might need to be updated to catch corner case, contact the repo maintainers." - exit 1 - fi + # Cleanup after SDK generation + cd "${SDK_REPO_LOCAL_PATH}" + goimports -w "${SERVICES_FOLDER}/" + make sync-tidy - contains_empty_space_pattern=" |'" - if [[ ${service_json} =~ ${contains_empty_space_pattern} ]]; then - echo "OAS filename ${service_json} has empty spaces, the generation will fail. If the OAS was downloaded using the make download-oas command, it should be fixed in the api-specifications repo, please contact the repo maintainers at ${OAS_REPO}." - exit 1 - fi + if [[ -n "$warning" ]]; then + echo -e "\nSome of the services were skipped during creation!\n$warning" + fi +} - # check if the whole service is blocklisted - if grep -E "^$service$" "${ROOT_DIR}/languages/golang/blocklist.txt"; then - echo "Skipping blocklisted service ${service}" - warning+="Skipping blocklisted service ${service}\n" - continue - fi - - - echo -e "\n>> Generating SDK for \"${service}\" service..." - for version_dir in "${service_dir}"/*; do - service_version_json="${version_dir}/${service_dir##*/}.json" - version="${version_dir##*/}" - - # check if that specific API version of the service is blocklisted - if grep -E "^${service}-${version}$" "${ROOT_DIR}/languages/golang/blocklist.txt"; then - echo "Skipping blocklisted API version ${version} of service ${service}" - warning+="Skipping blocklisted API version ${version} of service ${service}\n" - continue - fi - - echo -e "\n>> Generating SDK package \"${version}api\" for \"${service}\" service..." - cd "${ROOT_DIR}" - - mkdir -p "${SERVICES_FOLDER}/${service}/${version}api" - cp "${ROOT_DIR}/languages/golang/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore" - - # Run the generator for Go - java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar:scripts/generators" \ - org.openapitools.codegen.OpenAPIGenerator generate \ - -g GoGenerator \ - --input-spec "${service_version_json}" \ - --output "${SERVICES_FOLDER}/${service}/${version}api" \ - --package-name "${version}api" \ - --enable-post-process-file \ - --git-host "${GIT_HOST}" \ - --git-user-id "${GIT_USER_ID}" \ - --git-repo-id "${GIT_REPO_ID}/services/${service}" \ - --global-property apis,models,modelTests=true,modelDocs=false,apiDocs=false,supportingFiles,apiTests=false\ - --inline-schema-options "SKIP_SCHEMA_REUSE=true,RESOLVE_INLINE_ENUMS=true" \ - --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ - --http-user-agent "stackit-sdk-go/${service}" \ - --reserved-words-mappings type=types \ - --config "${ROOT_DIR}/languages/golang/openapi-generator-config.yml" - - # Remove unnecessary files - rm "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore" - rm -r "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator" - - # If the service version has a wait package files, move them inside the service folder - if [ -d "${sdk_services_backup_dir}/${service}/${version}api/wait" ]; then - echo "Found ${service} \"wait\" package" - cp -r "${sdk_services_backup_dir}/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/${version}api/wait" - fi - - # If the service version has a handwritten.go file, move them inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/${version}api/handwritten.go" ]; then - echo "Found ${service} \"handwritten.go\" file" - cp -r "${sdk_services_backup_dir}/${service}/${version}api/handwritten.go" "${SERVICES_FOLDER}/${service}/${version}api/handwritten.go" - fi - done - - if ! grep -E "^$service$" "${ROOT_DIR}/languages/golang/compat-layer/allow-list.txt"; then - echo "Skipping service ${service}, compatibility layer is not activated for it" - warning+="Skipping compatibility layer generation for service ${service}\n" - - if [ ! -f "${SERVICES_FOLDER}/${service}/go.mod" ]; then - printf "module ${GIT_HOST}/${GIT_USER_ID}/${GIT_REPO_ID}/services/${service}\n\n" > "${SERVICES_FOLDER}/${service}/go.mod" - printf "go ${SDK_GO_VERSION}\n\n" >> "${SERVICES_FOLDER}/${service}/go.mod" - printf "require (\n\tgithub.com/stackitcloud/stackit-sdk-go/core v0.21.1\n)\n" >> "${SERVICES_FOLDER}/${service}/go.mod" - fi - - # generate package.go - printf "package ${service}\n" > "${SERVICES_FOLDER}/${service}/package.go" - - # If the service has a LICENSE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then - echo "Found ${service} \"LICENSE\" file" - cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" - else - cp "${ROOT_DIR}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" - fi - - # If the service has a CHANGELOG file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then - echo "Found ${service} \"CHANGELOG\" file" - cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" - fi - - # If the service has a NOTICE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then - echo "Found ${service} \"NOTICE\" file" - cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" - fi - - # If the service has a VERSION file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/VERSION" ]; then - echo "Found ${service} \"VERSION\" file" - cp -r "${sdk_services_backup_dir}/${service}/VERSION" "${SERVICES_FOLDER}/${service}/VERSION" - fi - - # If the service has oas_commit file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then - echo "Found ${service} \"oas_commit\" file" - cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" - fi - - cd "${SERVICES_FOLDER}/${service}" - go work use . - # Make sure that dependencies are uptodate - go get -u ./... - go mod tidy +generate_go_service() { + local service_dir=$1 + local service=$2 + local compat_layer_service_oas_name=$3 + + echo -e "\n>> Generating SDK for \"${service}\" service..." + for version_dir in "${service_dir}"/*; do + service_version_json="${version_dir}/${service_dir##*/}.json" + version="${version_dir##*/}" + # check if that specific API version of the service is blocklisted + if grep -E "^${service}-${version}$" "${ROOT_DIR}/languages/golang/blocklist.txt"; then + echo "Skipping blocklisted API version ${version} of service ${service}" + warning+="Skipping blocklisted API version ${version} of service ${service}\n" continue fi - - # COMPAT LAYER - LEGACY !! - START - - # Download OpenAPI generator if not already downloaded - compat_layer_jar_path="${ROOT_DIR}/scripts/bin/openapi-generator-cli-go-compat-layer.jar" - if [ -e ${compat_layer_jar_path} ] && [ $(java -jar ${compat_layer_jar_path} version) == "6.6.0" ]; then - : - else - echo "Downloading OpenAPI generator (version 6.6.0) for generating the compatibility layer..." - mkdir -p "${ROOT_DIR}/scripts/bin" - wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.6.0/openapi-generator-cli-6.6.0.jar -O ${compat_layer_jar_path} --quiet - echo "Download done." - fi - echo -e "\n>> Generating compatibility layer for \"${service}\" service..." + echo -e "\n>> Generating SDK package \"${version}api\" for \"${service}\" service..." cd "${ROOT_DIR}" - mkdir -p "${SERVICES_FOLDER}/${service}" - cp "${ROOT_DIR}/languages/golang/compat-layer/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - regional_api= - if grep -E "^$service$" ${ROOT_DIR}/languages/golang/compat-layer/regional-allowlist.txt; then - echo "Generating new regional api" - regional_api="regional_api" - fi - - # Run the compatibility-layer generator for Go - java -Dlog.level=${GENERATOR_LOG_LEVEL} -jar ${compat_layer_jar_path} generate \ - --generator-name go \ - --input-spec "${ROOT_DIR}/oas/legacy/${compat_layer_service_oas_name}.json" \ - --output "${SERVICES_FOLDER}/${service}" \ - --package-name "${service}" \ + mkdir -p "${SERVICES_FOLDER}/${service}/${version}api" + cp "${ROOT_DIR}/languages/golang/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore" + + # Run the generator for Go + java -Dlog.level=${GENERATOR_LOG_LEVEL} -cp "custom:scripts/bin/openapi-generator-cli.jar:scripts/generators" \ + org.openapitools.codegen.OpenAPIGenerator generate \ + -g GoGenerator \ + --input-spec "${service_version_json}" \ + --output "${SERVICES_FOLDER}/${service}/${version}api" \ + --package-name "${version}api" \ --enable-post-process-file \ --git-host "${GIT_HOST}" \ --git-user-id "${GIT_USER_ID}" \ - --git-repo-id "${GIT_REPO_ID}" \ - --global-property apis,models,modelTests=true,modelDocs=false,apiDocs=false,supportingFiles,apiTests=false \ - --additional-properties=isGoSubmodule=true,enumClassPrefix=true,generateInterfaces=true,$regional_api \ + --git-repo-id "${GIT_REPO_ID}/services/${service}" \ + --global-property apis,models,modelTests=true,modelDocs=false,apiDocs=false,supportingFiles,apiTests=false\ + --inline-schema-options "SKIP_SCHEMA_REUSE=true,RESOLVE_INLINE_ENUMS=true" \ + --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ --http-user-agent "stackit-sdk-go/${service}" \ --reserved-words-mappings type=types \ - --config "${ROOT_DIR}/languages/golang/compat-layer/openapi-generator-config.yml" - + --config "${ROOT_DIR}/languages/golang/openapi-generator-config.yml" + # Remove unnecessary files - rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - rm "${SERVICES_FOLDER}/${service}/.openapi-generator/FILES" - - # If there's a comment at the start of go.mod, copy it - go_mod_backup_path="${sdk_services_backup_dir}/${service}/go.mod" - if [ -f ${go_mod_backup_path} ]; then - go_mod_backup_first_line="$(head -n 1 ${go_mod_backup_path})" - is_comment_pattern="^\/\/" - if [[ ${go_mod_backup_first_line} =~ ${is_comment_pattern} ]]; then - echo "Found comment at the top of ${service}/go.mod" - go_mod_path="${SERVICES_FOLDER}/${service}/go.mod" - echo -e "${go_mod_backup_first_line}\n$(cat ${go_mod_path})" >${go_mod_path} - fi - fi + rm "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator-ignore" + rm -r "${SERVICES_FOLDER}/${service}/${version}api/.openapi-generator" - # If the service has a wait package files, move them inside the service folder - if [ -d "${sdk_services_backup_dir}/${service}/wait" ]; then + # If the service version has a wait package files, move them inside the service folder + if [ -d "${sdk_services_backup_dir}/${service}/${version}api/wait" ]; then echo "Found ${service} \"wait\" package" - cp -r "${sdk_services_backup_dir}/${service}/wait" "${SERVICES_FOLDER}/${service}/wait" - # deprecate legacy wait package - printf "// Deprecated: Will be removed after 2026-09-30. Move to the packages generated for each available API version instead\npackage wait\n\n" > "${SERVICES_FOLDER}/${service}/wait/deprecation.go" + cp -r "${sdk_services_backup_dir}/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/${version}api/wait" fi - # If the service has a CHANGELOG file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then - echo "Found ${service} \"CHANGELOG\" file" - cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" + # If the service version has a handwritten.go file, move them inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/${version}api/handwritten.go" ]; then + echo "Found ${service} \"handwritten.go\" file" + cp -r "${sdk_services_backup_dir}/${service}/${version}api/handwritten.go" "${SERVICES_FOLDER}/${service}/${version}api/handwritten.go" + fi + done + + if ! grep -E "^$service$" "${ROOT_DIR}/languages/golang/compat-layer/allow-list.txt"; then + echo "Skipping service ${service}, compatibility layer is not activated for it" + warning+="Skipping compatibility layer generation for service ${service}\n" + + if [ ! -f "${SERVICES_FOLDER}/${service}/go.mod" ]; then + printf "module ${GIT_HOST}/${GIT_USER_ID}/${GIT_REPO_ID}/services/${service}\n\n" > "${SERVICES_FOLDER}/${service}/go.mod" + printf "go ${SDK_GO_VERSION}\n\n" >> "${SERVICES_FOLDER}/${service}/go.mod" + printf "require (\n\tgithub.com/stackitcloud/stackit-sdk-go/core v0.21.1\n)\n" >> "${SERVICES_FOLDER}/${service}/go.mod" fi + # generate package.go + printf "package ${service}\n" > "${SERVICES_FOLDER}/${service}/package.go" + # If the service has a LICENSE file, move it inside the service folder if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then echo "Found ${service} \"LICENSE\" file" cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" + else + cp "${ROOT_DIR}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" + fi + + # If the service has a CHANGELOG file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then + echo "Found ${service} \"CHANGELOG\" file" + cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" fi # If the service has a NOTICE file, move it inside the service folder @@ -347,37 +253,115 @@ generate_go_sdk() { echo "Found ${service} \"oas_commit\" file" cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" fi - + cd "${SERVICES_FOLDER}/${service}" go work use . # Make sure that dependencies are uptodate go get -u ./... go mod tidy - - # COMPAT LAYER - LEGACY !! - END - done + return + fi - # Add examples to workspace - if [ -d "${EXAMPLES_FOLDER}" ]; then - for example_dir in ${EXAMPLES_FOLDER}/*; do - cd "${example_dir}" - go work use . - done + # COMPAT LAYER - LEGACY !! - START + + # Download OpenAPI generator if not already downloaded + compat_layer_jar_path="${ROOT_DIR}/scripts/bin/openapi-generator-cli-go-compat-layer.jar" + if [ -e ${compat_layer_jar_path} ] && [ $(java -jar ${compat_layer_jar_path} version) == "6.6.0" ]; then + : + else + echo "Downloading OpenAPI generator (version 6.6.0) for generating the compatibility layer..." + mkdir -p "${ROOT_DIR}/scripts/bin" + wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.6.0/openapi-generator-cli-6.6.0.jar -O ${compat_layer_jar_path} --quiet + echo "Download done." fi - # Add scripts to workspace - if [ -d "${SCRIPTS_FOLDER}" ]; then - cd "${SCRIPTS_FOLDER}" - go work use . + echo -e "\n>> Generating compatibility layer for \"${service}\" service..." + cd "${ROOT_DIR}" + + mkdir -p "${SERVICES_FOLDER}/${service}" + cp "${ROOT_DIR}/languages/golang/compat-layer/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + regional_api= + if grep -E "^$service$" ${ROOT_DIR}/languages/golang/compat-layer/regional-allowlist.txt; then + echo "Generating new regional api" + regional_api="regional_api" fi - # Cleanup after SDK generation - cd "${SDK_REPO_LOCAL_PATH}" - goimports -w "${SERVICES_FOLDER}/" - make sync-tidy + # Run the compatibility-layer generator for Go + java -Dlog.level=${GENERATOR_LOG_LEVEL} -jar ${compat_layer_jar_path} generate \ + --generator-name go \ + --input-spec "${ROOT_DIR}/oas/legacy/${compat_layer_service_oas_name}.json" \ + --output "${SERVICES_FOLDER}/${service}" \ + --package-name "${service}" \ + --enable-post-process-file \ + --git-host "${GIT_HOST}" \ + --git-user-id "${GIT_USER_ID}" \ + --git-repo-id "${GIT_REPO_ID}" \ + --global-property apis,models,modelTests=true,modelDocs=false,apiDocs=false,supportingFiles,apiTests=false \ + --additional-properties=isGoSubmodule=true,enumClassPrefix=true,generateInterfaces=true,$regional_api \ + --http-user-agent "stackit-sdk-go/${service}" \ + --reserved-words-mappings type=types \ + --config "${ROOT_DIR}/languages/golang/compat-layer/openapi-generator-config.yml" + + # Remove unnecessary files + rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + rm "${SERVICES_FOLDER}/${service}/.openapi-generator/FILES" + + # If there's a comment at the start of go.mod, copy it + go_mod_backup_path="${sdk_services_backup_dir}/${service}/go.mod" + if [ -f ${go_mod_backup_path} ]; then + go_mod_backup_first_line="$(head -n 1 ${go_mod_backup_path})" + is_comment_pattern="^\/\/" + if [[ ${go_mod_backup_first_line} =~ ${is_comment_pattern} ]]; then + echo "Found comment at the top of ${service}/go.mod" + go_mod_path="${SERVICES_FOLDER}/${service}/go.mod" + echo -e "${go_mod_backup_first_line}\n$(cat ${go_mod_path})" >${go_mod_path} + fi + fi - if [[ -n "$warning" ]]; then - echo -e "\nSome of the services were skipped during creation!\n$warning" + # If the service has a wait package files, move them inside the service folder + if [ -d "${sdk_services_backup_dir}/${service}/wait" ]; then + echo "Found ${service} \"wait\" package" + cp -r "${sdk_services_backup_dir}/${service}/wait" "${SERVICES_FOLDER}/${service}/wait" + # deprecate legacy wait package + printf "// Deprecated: Will be removed after 2026-09-30. Move to the packages generated for each available API version instead\npackage wait\n\n" > "${SERVICES_FOLDER}/${service}/wait/deprecation.go" + fi + + # If the service has a CHANGELOG file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then + echo "Found ${service} \"CHANGELOG\" file" + cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" + fi + + # If the service has a LICENSE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then + echo "Found ${service} \"LICENSE\" file" + cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" fi + + # If the service has a NOTICE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then + echo "Found ${service} \"NOTICE\" file" + cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" + fi + + # If the service has a VERSION file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/VERSION" ]; then + echo "Found ${service} \"VERSION\" file" + cp -r "${sdk_services_backup_dir}/${service}/VERSION" "${SERVICES_FOLDER}/${service}/VERSION" + fi + + # If the service has oas_commit file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then + echo "Found ${service} \"oas_commit\" file" + cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" + fi + + cd "${SERVICES_FOLDER}/${service}" + go work use . + # Make sure that dependencies are uptodate + go get -u ./... + go mod tidy + + # COMPAT LAYER - LEGACY !! - END } diff --git a/scripts/generate-sdk/languages/java.sh b/scripts/generate-sdk/languages/java.sh index 1d4ddc17..5377d6ac 100644 --- a/scripts/generate-sdk/languages/java.sh +++ b/scripts/generate-sdk/languages/java.sh @@ -57,6 +57,20 @@ generate_java_sdk() { fi git clone --quiet -b "${SDK_BRANCH}" "${SDK_REPO_URL}" "${SDK_REPO_LOCAL_PATH}" + plan_include_args=() + for service in "${INCLUDE_SERVICES[@]}"; do + plan_include_args+=(--include-service "${service}") + done + ( + cd "${ROOT_DIR}" + "${ROOT_DIR}/scripts/bin/build" --language java plan \ + --spec-dir "oas/services" \ + --service-dir "sdk-repo-updated/services" \ + --blocklist "languages/java/blocklist.txt" \ + --output "generation-plan.json" \ + "${plan_include_args[@]}" + ) + # Backup of the current state of the SDK services dir (services/) sdk_services_backup_dir=$(mktemp -d) if [[ ! "${sdk_services_backup_dir}" || -d {sdk_services_backup_dir} ]]; then @@ -81,139 +95,15 @@ generate_java_sdk() { javac -cp "${GENERATOR_JAR_PATH}" -d custom $(find ./scripts/generators -type f -name '*.java' | grep -v overrides) warning="" - - # Generate SDK for each service - for service_dir in "${ROOT_DIR}/oas/services"/*; do - service="${service_dir##*/}" - service="${service%.json}" - - service_pascal_case=$(to_pascal_case "${service}") - - # Remove invalid characters to ensure a valid Java pkg name - service="${service//-/}" # remove dashes - service="${service// /}" # remove spaces - service=$(echo "${service}" | tr '[:upper:]' '[:lower:]') # convert upper case letters to lower case - service=$(echo "${service}" | tr -d -c '[:alnum:]') # remove non-alphanumeric characters - - # Ensure the package name doesn't start with a number - if [[ "${service}" =~ ^[0-9] ]]; then - service="_${service}" # Prepend a valid prefix if it starts with a number - fi - - if ! [[ ${INCLUDE_SERVICES[*]} =~ ${service} ]]; then - echo "Skipping not included service ${service}" - warning+="Skipping not included service ${service}\n" - continue - fi - # check if the whole service is blocklisted - if grep -E "^$service$" "${ROOT_DIR}/languages/java/blocklist.txt"; then - echo "Skipping blocklisted service ${service}" - warning+="Skipping blocklisted service ${service}\n" - continue - fi - - echo -e "\n>> Generating SDK for \"${service}\" service..." - for version_dir in "${service_dir}"/*; do - service_version_json="${version_dir}/${service_dir##*/}.json" - version="${version_dir##*/}" - - # check if that specific API version of the service is blocklisted - if grep -E "^${service}-${version}$" "${ROOT_DIR}/languages/java/blocklist.txt"; then - echo "Skipping blocklisted API version ${version} of service ${service}" - warning+="Skipping blocklisted API version ${version} of service ${service}\n" - continue - fi - - echo -e "\n>> Generating SDK package \"${version}api\" for \"${service}\" service..." - cd "${ROOT_DIR}" - - mkdir -p "${SERVICES_FOLDER}/${service}" - cp "${ROOT_DIR}/languages/java/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - - SERVICE_DESCRIPTION=$(cat "${service_version_json}" | jq .info.title --raw-output) - - # Run the generator - java -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \ - org.openapitools.codegen.OpenAPIGenerator generate \ - --generator-name JavaGenerator \ - --input-spec "${service_version_json}" \ - --output "${SERVICES_FOLDER}/${service}" \ - --git-host "${GIT_HOST}" \ - --git-user-id "${GIT_USER_ID}" \ - --git-repo-id "${GIT_REPO_ID}" \ - --enable-post-process-file \ - --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=true,supportingFiles \ - --additional-properties="artifactId=${service},artifactDescription=${SERVICE_DESCRIPTION},invokerPackage=cloud.stackit.sdk.${service}.${version}api,modelPackage=cloud.stackit.sdk.${service}.${version}api.model,apiPackage=cloud.stackit.sdk.${service}.${version}api.api,serviceName=${service_pascal_case}" \ - --inline-schema-options "SKIP_SCHEMA_REUSE=true" \ - --http-user-agent stackit-sdk-java/"${service}" \ - --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ - --config "${ROOT_DIR}/languages/java/openapi-generator-config.yml" - - # Rename DefaultApiServiceApi.java to {serviceName}Api.java - # This approach is a workaround because the file name cannot be set dynamically via --additional-properties or the config file in OpenAPI Generator. - api_file="${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/api/DefaultApiServiceApi.java" - if [ -f "$api_file" ]; then - mv "$api_file" "${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/api/${service_pascal_case}Api.java" - fi - api_test_file="${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/api/DefaultApiTestServiceApiTest.java" - if [ -f "$api_test_file" ]; then - mv "$api_test_file" "${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/api/${service_pascal_case}ApiTest.java" - fi - - build_gradle="${SERVICES_FOLDER}/${service}/${version}api/build.gradle" - if [ -f "$build_gradle" ]; then - mv "$build_gradle" "${SERVICES_FOLDER}/${service}/build.gradle" - fi - - # Remove unnecessary files - rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/" - - # If the service version has a wait package, move them inside the service folder - if [ -d "${sdk_services_backup_dir}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" ]; then - echo "Found ${service} \"wait\" package" - cp -r "${sdk_services_backup_dir}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" - fi - - # If the service version has a wait test package, move them inside the service folder - if [ -d "${sdk_services_backup_dir}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" ]; then - echo "Found ${service} \"wait\" test package" - cp -r "${sdk_services_backup_dir}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" - fi - done - - # If the service has a CHANGELOG file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then - echo "Found ${service} \"CHANGELOG\" file" - cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" - fi - - # If the service has a LICENSE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then - echo "Found ${service} \"LICENSE\" file" - cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" - fi - - # If the service has a NOTICE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then - echo "Found ${service} \"NOTICE\" file" - cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" - fi - - # If the service has a VERSION file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/VERSION" ]; then - echo "Found ${service} \"VERSION\" file" - cp -r "${sdk_services_backup_dir}/${service}/VERSION" "${SERVICES_FOLDER}/${service}/VERSION" - fi - - # If the service has oas_commit file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then - echo "Found ${service} \"oas_commit\" file" - cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" - fi - - done + while IFS= read -r -d '' oas_service && IFS= read -r service; do + generate_java_service \ + "${ROOT_DIR}/oas/services/${oas_service}" \ + "${service}" \ + "$(to_pascal_case "${oas_service}")" + done < <( + "${ROOT_DIR}/scripts/bin/build" generate --plan "generation-plan.json" + ) cd "${SDK_REPO_LOCAL_PATH}" make fmt @@ -234,3 +124,109 @@ to_pascal_case() { print "" }' } + +generate_java_service(){ + local service_dir=$1 + local service=$2 + local service_pascal_case=$3 + + echo -e "\n>> Generating SDK for \"${service}\" service..." + for version_dir in "${service_dir}"/*; do + service_version_json="${version_dir}/${service_dir##*/}.json" + version="${version_dir##*/}" + + # check if that specific API version of the service is blocklisted + if grep -E "^${service}-${version}$" "${ROOT_DIR}/languages/java/blocklist.txt"; then + echo "Skipping blocklisted API version ${version} of service ${service}" + warning+="Skipping blocklisted API version ${version} of service ${service}\n" + continue + fi + + echo -e "\n>> Generating SDK package \"${version}api\" for \"${service}\" service..." + cd "${ROOT_DIR}" + + mkdir -p "${SERVICES_FOLDER}/${service}" + cp "${ROOT_DIR}/languages/java/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + + SERVICE_DESCRIPTION=$(cat "${service_version_json}" | jq .info.title --raw-output) + + # Run the generator + java -Dlog.level="${GENERATOR_LOG_LEVEL}" -cp "custom:${GENERATOR_JAR_PATH}:scripts/generators" \ + org.openapitools.codegen.OpenAPIGenerator generate \ + --generator-name JavaGenerator \ + --input-spec "${service_version_json}" \ + --output "${SERVICES_FOLDER}/${service}" \ + --git-host "${GIT_HOST}" \ + --git-user-id "${GIT_USER_ID}" \ + --git-repo-id "${GIT_REPO_ID}" \ + --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ + --enable-post-process-file \ + --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=true,supportingFiles \ + --additional-properties="artifactId=${service},artifactDescription=${SERVICE_DESCRIPTION},invokerPackage=cloud.stackit.sdk.${service}.${version}api,modelPackage=cloud.stackit.sdk.${service}.${version}api.model,apiPackage=cloud.stackit.sdk.${service}.${version}api.api,serviceName=${service_pascal_case}" \ + --inline-schema-options "SKIP_SCHEMA_REUSE=true" \ + --http-user-agent stackit-sdk-java/"${service}" \ + --config "${ROOT_DIR}/languages/java/openapi-generator-config.yml" + + # Rename DefaultApiServiceApi.java to {serviceName}Api.java + # This approach is a workaround because the file name cannot be set dynamically via --additional-properties or the config file in OpenAPI Generator. + api_file="${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/api/DefaultApiServiceApi.java" + if [ -f "$api_file" ]; then + mv "$api_file" "${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/api/${service_pascal_case}Api.java" + fi + api_test_file="${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/api/DefaultApiTestServiceApiTest.java" + if [ -f "$api_test_file" ]; then + mv "$api_test_file" "${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/api/${service_pascal_case}ApiTest.java" + fi + + build_gradle="${SERVICES_FOLDER}/${service}/${version}api/build.gradle" + if [ -f "$build_gradle" ]; then + mv "$build_gradle" "${SERVICES_FOLDER}/${service}/build.gradle" + fi + + # Remove unnecessary files + rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/" + + # If the service version has a wait package, move them inside the service folder + if [ -d "${sdk_services_backup_dir}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" ]; then + echo "Found ${service} \"wait\" package" + cp -r "${sdk_services_backup_dir}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/src/main/java/cloud/stackit/sdk/${service}/${version}api/wait" + fi + + # If the service version has a wait test package, move them inside the service folder + if [ -d "${sdk_services_backup_dir}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" ]; then + echo "Found ${service} \"wait\" test package" + cp -r "${sdk_services_backup_dir}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" "${SERVICES_FOLDER}/${service}/src/test/java/cloud/stackit/sdk/${service}/${version}api/wait" + fi + done + + # If the service has a CHANGELOG file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then + echo "Found ${service} \"CHANGELOG\" file" + cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" + fi + + # If the service has a LICENSE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then + echo "Found ${service} \"LICENSE\" file" + cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" + fi + + # If the service has a NOTICE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then + echo "Found ${service} \"NOTICE\" file" + cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" + fi + + # If the service has a VERSION file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/VERSION" ]; then + echo "Found ${service} \"VERSION\" file" + cp -r "${sdk_services_backup_dir}/${service}/VERSION" "${SERVICES_FOLDER}/${service}/VERSION" + fi + + # If the service has oas_commit file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then + echo "Found ${service} \"oas_commit\" file" + cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" + fi +} diff --git a/scripts/generate-sdk/languages/python.sh b/scripts/generate-sdk/languages/python.sh index b00f4a2a..a4ef4ee3 100644 --- a/scripts/generate-sdk/languages/python.sh +++ b/scripts/generate-sdk/languages/python.sh @@ -6,8 +6,6 @@ set -eo pipefail ROOT_DIR=$(git rev-parse --show-toplevel) SDK_REPO_LOCAL_PATH="${ROOT_DIR}/sdk-repo-updated" -OAS_REPO=https://github.com/stackitcloud/stackit-api-specifications - SERVICES_FOLDER="${SDK_REPO_LOCAL_PATH}/services" GENERATOR_LOG_LEVEL="error" # Must be a Java log level (error, warn, info...) @@ -57,6 +55,15 @@ generate_python_sdk() { fi git clone --quiet -b "${SDK_BRANCH}" "${SDK_REPO_URL}" "${SDK_REPO_LOCAL_PATH}" + ( + cd "${ROOT_DIR}" + "${ROOT_DIR}/scripts/bin/build" --language python plan \ + --spec-dir "oas/services" \ + --service-dir "sdk-repo-updated/services" \ + --blocklist "languages/python/blocklist.txt" \ + --output "generation-plan.json" + ) + # Install SDK project tools cd "${ROOT_DIR}" make project-tools LANGUAGE=python @@ -78,8 +85,6 @@ generate_python_sdk() { # Remove old contents of services dir (services/) rm -rf "${SERVICES_FOLDER}" - warning="" - # There is no core in the internal python sdk, therefore the generation of pyproject.toml # needs some adjustment. local extra_props_internal="useLocalCore=true" @@ -90,114 +95,106 @@ generate_python_sdk() { # TODO: add to generator below when adding multi-API-version support: # --inline-schema-options "SKIP_SCHEMA_REUSE=true" - # Generate SDK for each service - for service_json in ${ROOT_DIR}/oas/legacy/*.json; do - service="${service_json##*/}" - service="${service%.json}" - - # Remove invalid characters to ensure a valid Go pkg name - service="${service//-/}" # remove dashes - service="${service// /}" # remove empty spaces - service="${service//_/}" # remove underscores - service=$(echo "${service}" | tr '[:upper:]' '[:lower:]') # convert upper case letters to lower case - service=$(echo "${service}" | tr -d -c '[:alnum:]') # remove non-alphanumeric characters - - if grep -E "^$service$" "${ROOT_DIR}/languages/python/blocklist.txt"; then - echo "Skipping blocklisted service ${service}" - warning+="Skipping blocklisted service ${service}\n" + while IFS= read -r -d '' oas_service && IFS= read -r service; do + local service_spec="${ROOT_DIR}/oas/legacy/${oas_service}.json" + if [[ ! -f "${service_spec}" ]]; then + echo "Skipping \"${service}\": legacy OpenAPI spec not found at ${service_spec}" continue fi - echo ">> Generating \"${service}\" service..." - cd "${ROOT_DIR}" - - mkdir -p "${SERVICES_FOLDER}/${service}/" - cp "${ROOT_DIR}/languages/python/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - - # Run the generator - java -Dlog.level="${GENERATOR_LOG_LEVEL}" -jar "${jar_path}" generate \ - --generator-name python \ - --input-spec "${service_json}" \ - --output "${SERVICES_FOLDER}/${service}" \ - --package-name "stackit.${service}" \ - --template-dir "${ROOT_DIR}/languages/python/templates/" \ - --git-host "${GIT_HOST}" \ - --git-user-id "${GIT_USER_ID}" \ - --git-repo-id "${GIT_REPO_ID}" \ - --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ - --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=false,supportingFiles \ - --additional-properties=pythonPackageName="stackit-${service},removeEnumValuePrefix=false,${extra_props_internal}" >/dev/null \ - --http-user-agent "stackit-sdk-python/${service}" - - # Remove unnecessary files - rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" - rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/" - rm "${SERVICES_FOLDER}/${service}/stackit/__init__.py" - rm "${SERVICES_FOLDER}/${service}/.github/workflows/python.yml" - - # Create source layout - mkdir "${SERVICES_FOLDER}/${service}/src" - mv "${SERVICES_FOLDER}/${service}/stackit/" "${SERVICES_FOLDER}/${service}/src/" - - # If the service has a wait package files, move them inside the service folder - if [ -d "${sdk_services_backup_dir}/${service}/src/wait" ]; then - echo "Found ${service} \"wait\" package" - cp -r "${sdk_services_backup_dir}/${service}/src/wait" "${SERVICES_FOLDER}/${service}/src/wait" - fi - - # If the service has a README.md file, move them inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/README.md" ]; then - echo "Found ${service} \"README.md\" file" - cp -r "${sdk_services_backup_dir}/${service}/README.md" "${SERVICES_FOLDER}/${service}/README.md" - fi + generate_python_service "${service_spec}" "${service}" + done < <( + "${ROOT_DIR}/scripts/bin/build" generate --plan "generation-plan.json" + ) +} - # If the service has a pyproject.toml file, move them inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/pyproject.toml" ]; then - echo "Found ${service} \"pyproject.toml\" file" - cp -r "${sdk_services_backup_dir}/${service}/pyproject.toml" "${SERVICES_FOLDER}/${service}/pyproject.toml" - fi +generate_python_service() { + local service_json=$1 + local service=$2 + echo ">> Generating \"${service}\" service..." + cd "${ROOT_DIR}" - # If the service has a uv.lock file, move them inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/uv.lock" ]; then - echo "Found ${service} \"uv.lock\" file" - cp -r "${sdk_services_backup_dir}/${service}/uv.lock" "${SERVICES_FOLDER}/${service}/uv.lock" - else - uv lock --directory "${SERVICES_FOLDER}/${service}" - fi + mkdir -p "${SERVICES_FOLDER}/${service}/" + cp "${ROOT_DIR}/languages/python/.openapi-generator-ignore" "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + + # Run the generator + java -Dlog.level="${GENERATOR_LOG_LEVEL}" -jar "${jar_path}" generate \ + --generator-name python \ + --input-spec "${service_json}" \ + --output "${SERVICES_FOLDER}/${service}" \ + --package-name "stackit.${service}" \ + --template-dir "${ROOT_DIR}/languages/python/templates/" \ + --git-host "${GIT_HOST}" \ + --git-user-id "${GIT_USER_ID}" \ + --git-repo-id "${GIT_REPO_ID}" \ + --openapi-normalizer "SIMPLIFY_ONEOF_ANYOF=false" \ + --global-property apis,models,modelTests=false,modelDocs=false,apiDocs=false,apiTests=false,supportingFiles \ + --additional-properties=pythonPackageName="stackit-${service},removeEnumValuePrefix=false,${extra_props_internal}" >/dev/null \ + --http-user-agent "stackit-sdk-python/${service}" + + # Remove unnecessary files + rm "${SERVICES_FOLDER}/${service}/.openapi-generator-ignore" + rm -r "${SERVICES_FOLDER}/${service}/.openapi-generator/" + rm "${SERVICES_FOLDER}/${service}/stackit/__init__.py" + rm "${SERVICES_FOLDER}/${service}/.github/workflows/python.yml" + + # Create source layout + mkdir "${SERVICES_FOLDER}/${service}/src" + mv "${SERVICES_FOLDER}/${service}/stackit/" "${SERVICES_FOLDER}/${service}/src/" + + # If the service has a wait package files, move them inside the service folder + if [ -d "${sdk_services_backup_dir}/${service}/src/wait" ]; then + echo "Found ${service} \"wait\" package" + cp -r "${sdk_services_backup_dir}/${service}/src/wait" "${SERVICES_FOLDER}/${service}/src/wait" + fi - # If the service has a CHANGELOG file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then - echo "Found ${service} \"CHANGELOG\" file" - cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" - fi + # If the service has a README.md file, move them inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/README.md" ]; then + echo "Found ${service} \"README.md\" file" + cp -r "${sdk_services_backup_dir}/${service}/README.md" "${SERVICES_FOLDER}/${service}/README.md" + fi - # If the service has a LICENSE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then - echo "Found ${service} \"LICENSE\" file" - cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" - fi + # If the service has a pyproject.toml file, move them inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/pyproject.toml" ]; then + echo "Found ${service} \"pyproject.toml\" file" + cp -r "${sdk_services_backup_dir}/${service}/pyproject.toml" "${SERVICES_FOLDER}/${service}/pyproject.toml" + fi - # If the service has a NOTICE file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then - echo "Found ${service} \"NOTICE\" file" - cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" - fi + # If the service has a uv.lock file, move them inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/uv.lock" ]; then + echo "Found ${service} \"uv.lock\" file" + cp -r "${sdk_services_backup_dir}/${service}/uv.lock" "${SERVICES_FOLDER}/${service}/uv.lock" + else + uv lock --directory "${SERVICES_FOLDER}/${service}" + fi - # If the service has oas_commit file, move it inside the service folder - if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then - echo "Found ${service} \"oas_commit\" file" - cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" - fi + # If the service has a CHANGELOG file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/CHANGELOG.md" ]; then + echo "Found ${service} \"CHANGELOG\" file" + cp -r "${sdk_services_backup_dir}/${service}/CHANGELOG.md" "${SERVICES_FOLDER}/${service}/CHANGELOG.md" + fi - cd "${SERVICES_FOLDER}/${service}" - # Run formatter - isort . - autoimport --ignore-init-modules . - uv run black --config pyproject.toml . + # If the service has a LICENSE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/LICENSE.md" ]; then + echo "Found ${service} \"LICENSE\" file" + cp -r "${sdk_services_backup_dir}/${service}/LICENSE.md" "${SERVICES_FOLDER}/${service}/LICENSE.md" + fi - done + # If the service has a NOTICE file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/NOTICE.txt" ]; then + echo "Found ${service} \"NOTICE\" file" + cp -r "${sdk_services_backup_dir}/${service}/NOTICE.txt" "${SERVICES_FOLDER}/${service}/NOTICE.txt" + fi - if [[ -n "$warning" ]]; then - echo -e "\nSome of the services were skipped during creation!\n$warning" + # If the service has oas_commit file, move it inside the service folder + if [ -f "${sdk_services_backup_dir}/${service}/oas_commit" ]; then + echo "Found ${service} \"oas_commit\" file" + cp -r "${sdk_services_backup_dir}/${service}/oas_commit" "${SERVICES_FOLDER}/${service}/oas_commit" fi + + cd "${SERVICES_FOLDER}/${service}" + # Run formatter + isort . + autoimport --ignore-init-modules . + uv run black --config pyproject.toml . } diff --git a/scripts/sdk-create-pr.sh b/scripts/sdk-create-pr.sh index 3ce58563..e437dc28 100755 --- a/scripts/sdk-create-pr.sh +++ b/scripts/sdk-create-pr.sh @@ -15,13 +15,6 @@ if [ $# -lt 2 ]; then exit 1 fi -if type -p go >/dev/null; then - : -else - echo "! Go not installed, unable to proceed." - exit 1 -fi - if [ ! -d "${SDK_REPO_LOCAL_PATH}" ]; then echo "! SDK to commit not found in root. Please run make generate-sdk" exit 1 @@ -40,6 +33,12 @@ else LANGUAGE=$4 fi +DELETE_PLAN="generation-plan.json" +if [[ ! -f "${ROOT_DIR}/${DELETE_PLAN}" ]]; then + echo "! Generation plan not found: ${DELETE_PLAN}" + exit 1 +fi + # Create temp directory to work on work_dir="$ROOT_DIR/temp" mkdir -p "${work_dir}" @@ -148,12 +147,79 @@ for service_path in ${work_dir}/sdk_to_push/services/*; do if [[ "$branch" != "main" ]]; then echo ">> Creating PR for $service" git commit -m "Generate $service" - git push origin "$branch" - echo -e "$COMMIT_INFO\nspec comparison: ${compare_link}" | gh pr create --title "Generator: Update SDK /services/$service" --body-file - --head "$branch" --base "main" + if [[ -z "${NO_PUSH+x}" ]]; then + git push origin "$branch" + echo -e "$COMMIT_INFO\nspec comparison: ${compare_link}" | gh pr create --title "Generator: Update SDK /services/$service" --body-file - --head "$branch" --base "main" + else + echo ">> NO_PUSH is set; skipping push and PR creation for $service" + fi else echo ">> Pushing changes for $service service..." git commit -m "Generate $service: $COMMIT_INFO" - git push origin "$branch" + if [[ -z "${NO_PUSH+x}" ]]; then + git push origin "$branch" + else + echo ">> NO_PUSH is set; skipping push for $service" + fi fi fi done + +# Deleted services are absent from sdk_to_push, so obtain them from the plan. +deleted_services="$( + cd "${ROOT_DIR}" && + "${ROOT_DIR}/scripts/bin/build" --language "${LANGUAGE}" delete --plan "${DELETE_PLAN}" +)" + +while IFS= read -r service; do + [[ -z "${service}" ]] && continue + + echo -e "\n>> Detected deleted $service service" + + # Each deletion gets its own branch, just like generated-service updates. + if [[ "${BRANCH_PREFIX}" != "main" ]]; then + git switch main + branch="${BRANCH_PREFIX}/${service}" + git switch -c "${branch}" + else + branch="${BRANCH_PREFIX}" + fi + + case "${LANGUAGE}" in + go) + rm -rf "services/${service}" "examples/${service}" + # Remove stale service and example modules from the Go workspace. + go work use -r . + go work sync + ;; + python|java) + rm -rf "services/${service}" "examples/${service}" + ;; + esac + + # Use -A so removed files, including Go workspace metadata, are staged. + git add -A + if git diff --cached --quiet; then + echo ">> No files to remove for $service service" + continue + fi + + if [[ "${branch}" != "main" ]]; then + echo ">> Creating removal PR for $service" + git commit -m "Remove $service: removed from API specifications" + if [[ -z "${NO_PUSH+x}" ]]; then + git push origin "${branch}" + echo -e "$COMMIT_INFO" | gh pr create --title "Generator: Remove SDK /services/$service" --body-file - --head "${branch}" --base "main" + else + echo ">> NO_PUSH is set; skipping push and PR creation for $service" + fi + else + echo ">> Pushing removal of $service service..." + git commit -m "Remove $service: $COMMIT_INFO" + if [[ -z "${NO_PUSH+x}" ]]; then + git push origin "${branch}" + else + echo ">> NO_PUSH is set; skipping push for $service" + fi + fi +done <<< "${deleted_services}"