diff --git a/.gitignore b/.gitignore index a163692..6135045 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,11 @@ import.html report*.html .load-data-done .env +.env.default +.env.local +.env.active +!.env.*.example +!.env.example state.json screenshots/ videos/ diff --git a/cortexapps_cli/commands/catalog.py b/cortexapps_cli/commands/catalog.py index dba4679..2717b89 100644 --- a/cortexapps_cli/commands/catalog.py +++ b/cortexapps_cli/commands/catalog.py @@ -65,149 +65,50 @@ class CatalogCommandOptions: typer.Option("--types", "-t", help="Filter the response to specific types of entities. By default, this includes services, resources, and domains. Corresponds to the x-cortex-type field in the Entity Descriptor.", show_default=False) ] -@app.command(name="list") -def catalog_list( - ctx: typer.Context, - include_archived: CatalogCommandOptions.include_archived = False, - hierarchy_depth: CatalogCommandOptions.hierarchy_depth = 'full', - groups: CatalogCommandOptions.groups = None, - owners: CatalogCommandOptions.owners = None, - include_hierarchy_fields: CatalogCommandOptions.include_hierarchy_fields = None, - include_nested_fields: CatalogCommandOptions.include_nested_fields = None, - include_owners: CatalogCommandOptions.include_owners = False, - include_links: CatalogCommandOptions.include_links = False, - include_metadata: CatalogCommandOptions.include_metadata = False, - git_repositories: CatalogCommandOptions.git_repositories = None, - types: CatalogCommandOptions.types = None, - page: ListCommandOptions.page = None, - page_size: ListCommandOptions.page_size = 250, - table_output: ListCommandOptions.table_output = False, - csv_output: ListCommandOptions.csv_output = False, - columns: ListCommandOptions.columns = [], - no_headers: ListCommandOptions.no_headers = False, - filters: ListCommandOptions.filters = [], - sort: ListCommandOptions.sort = [], - _print: CommandOptions._print = True, -): - """ - List entities in the catalog - """ - client = ctx.obj["client"] - - if (table_output or csv_output) and not ctx.params.get('columns'): - ctx.params['columns'] = [ - "ID=id", - "Tag=tag", - "Name=name", - "Type=type", - "Git Repository=git.repository", - ] - - params = { - "includeArchived": include_archived, - "hierarchyDepth": hierarchy_depth, - "groups": groups, - "owners": owners, - "includeHierarchyFields": include_hierarchy_fields, - "includeNestedFields": include_nested_fields, - "includeOwners": include_owners, - "includeLinks": include_links, - "includeMetadata": include_metadata, - "page": page, - "pageSize": page_size, - "gitRepositories": git_repositories, - "types": types, - } - - # remove any params that are None - params = {k: v for k, v in params.items() if v is not None} - - # for keys that can have multiple values, remove whitespace around comma and split on comma - for key in ['groups', 'owners', 'gitRepositories', 'types']: - if key in params: - params[key] = [x.strip() for x in params[key].split(',')] - - if page is None: - # if page is not specified, we want to fetch all pages - r = client.fetch("api/v1/catalog", params=params) - else: - # if page is specified, we want to fetch only that page - r = client.get("api/v1/catalog", params=params) - - if _print: - data = r - print_output_with_context(ctx, data) - else: - return(r) - @app.command() -def details( +def archive( ctx: typer.Context, - hierarchy_depth: CatalogCommandOptions.hierarchy_depth = 'full', - include_hierarchy_fields: CatalogCommandOptions.include_hierarchy_fields = None, tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), - table_output: ListCommandOptions.table_output = False, - csv_output: ListCommandOptions.csv_output = False, - no_headers: ListCommandOptions.no_headers = False, - columns: ListCommandOptions.columns = [], - filters: ListCommandOptions.filters = [], ): """ - Get details for a specific entity in the catalog + Archive an entity """ client = ctx.obj["client"] - if table_output and csv_output: - raise typer.BadParameter("Only one of --table and --csv can be specified") - - if (table_output or csv_output) and not ctx.params.get('columns'): - ctx.params['columns'] = [ - "ID=id", - "Tag=tag", - "Name=name", - "Type=type", - "Git Repository=git.repository", - ] - - output_format = "table" if table_output else "csv" if csv_output else "json" - - params = { - "hierarchyDepth": hierarchy_depth, - "includeHierarchyFields": include_hierarchy_fields - } - - # remove any params that are None - params = {k: v for k, v in params.items() if v is not None} - - r = client.get("api/v1/catalog/" + tag, params=params) - - data = r if output_format == 'json' else [r] - print_output_with_context(ctx, data) + r = client.put("api/v1/catalog/" + tag + "/archive") @app.command() -def archive( +def aws( ctx: typer.Context, tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), ): """ - Archive an entity + Get AWS resource details for an entity """ client = ctx.obj["client"] - r = client.put("api/v1/catalog/" + tag + "/archive") + r = client.get("api/v1/catalog/" + tag + "/aws") + print_output_with_context(ctx, r) @app.command() -def unarchive( +def create( ctx: typer.Context, - tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), + file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help=" File containing YAML content of entity; can be passed as stdin with -, example: -f-")] = None, + dry_run: CatalogCommandOptions.dry_run = False, + _print: CommandOptions._print = True, ): """ - Unarchive an entity + Create entity """ client = ctx.obj["client"] - r = client.put("api/v1/catalog/" + tag + "/unarchive") - print_output_with_context(ctx, r) + params = { + "dryRun": dry_run + } + + r = client.post("api/v1/open-api", data=file_input.read(), params=params, content_type="application/openapi;charset=UTF-8") + if _print: + print_output_with_context(ctx, r) @app.command() def delete( @@ -239,7 +140,6 @@ def delete_by_type( client.delete("api/v1/catalog", params=params) - @app.command() def descriptor( ctx: typer.Context, @@ -270,48 +170,166 @@ def descriptor( print_output_with_context(ctx, r) @app.command() -def create( +def details( ctx: typer.Context, - file_input: Annotated[typer.FileText, typer.Option("--file", "-f", help=" File containing YAML content of entity; can be passed as stdin with -, example: -f-")] = None, - dry_run: CatalogCommandOptions.dry_run = False, - _print: CommandOptions._print = True, + hierarchy_depth: CatalogCommandOptions.hierarchy_depth = 'full', + include_hierarchy_fields: CatalogCommandOptions.include_hierarchy_fields = None, + tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), + table_output: ListCommandOptions.table_output = False, + csv_output: ListCommandOptions.csv_output = False, + no_headers: ListCommandOptions.no_headers = False, + columns: ListCommandOptions.columns = [], + filters: ListCommandOptions.filters = [], ): """ - Create entity + Get details for a specific entity in the catalog """ client = ctx.obj["client"] + if table_output and csv_output: + raise typer.BadParameter("Only one of --table and --csv can be specified") + + if (table_output or csv_output) and not ctx.params.get('columns'): + ctx.params['columns'] = [ + "ID=id", + "Tag=tag", + "Name=name", + "Type=type", + "Git Repository=git.repository", + ] + + output_format = "table" if table_output else "csv" if csv_output else "json" + params = { - "dryRun": dry_run + "hierarchyDepth": hierarchy_depth, + "includeHierarchyFields": include_hierarchy_fields } - r = client.post("api/v1/open-api", data=file_input.read(), params=params, content_type="application/openapi;charset=UTF-8") - if _print: - print_output_with_context(ctx, r) + # remove any params that are None + params = {k: v for k, v in params.items() if v is not None} + + r = client.get("api/v1/catalog/" + tag, params=params) + + data = r if output_format == 'json' else [r] + print_output_with_context(ctx, data) @app.command() -def patch( +def gitops_log( ctx: typer.Context, - file_input: Annotated[typer.FileText, typer.Option(..., "--file", "-f", help=" File containing YAML content of entity; can be passed as stdin with -, example: -f-")] = None, - delete_marker_value = typer.Option("__delete__", "--delete-marker-value", "-dmv", help="Delete keys with this value from the merged yaml, defaults to __delete__, if any values match this, they will not be included in merged YAML. For example my_value: __delete__ will remove my_value from the merged YAML."), - dry_run: CatalogCommandOptions.dry_run = False, - append_arrays: CatalogCommandOptions.append_arrays = False, - fail_if_not_exist: CatalogCommandOptions.fail_if_not_exist = False, + tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), ): """ - Creates or updates an entity. If the YAML refers to an entity that already exists (as referenced by the x-cortex-tag), this API will merge the specified changes into the existing entity + Retrieve most recent GitOps log for entity + """ + client = ctx.obj["client"] + + r = client.get("api/v1/catalog/" + tag + "/gitops-logs") + print_output_with_context(ctx, r) + +@app.command() +def k8s( + ctx: typer.Context, + tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), + table_output: ListCommandOptions.table_output = False, + csv_output: ListCommandOptions.csv_output = False, + no_headers: ListCommandOptions.no_headers = False, + columns: ListCommandOptions.columns = [], + filters: ListCommandOptions.filters = [], +): + """ + Get Kubernetes resource details for an entity + """ + client = ctx.obj["client"] + + if table_output and csv_output: + raise typer.BadParameter("Only one of --table and --csv can be specified") + + if (table_output or csv_output) and not ctx.params.get('columns'): + ctx.params['columns'] = [ + "Namespace=namespace", + "Name=name", + "Cluster=cluster", + "Type=type", + "Last Updated=lastUpdated", + ] + + r = client.get("api/v1/catalog/" + tag + "/k8s") + print_output_with_context(ctx, r) + +@app.command(name="list") +def catalog_list( + ctx: typer.Context, + include_archived: CatalogCommandOptions.include_archived = False, + hierarchy_depth: CatalogCommandOptions.hierarchy_depth = 'full', + groups: CatalogCommandOptions.groups = None, + owners: CatalogCommandOptions.owners = None, + include_hierarchy_fields: CatalogCommandOptions.include_hierarchy_fields = None, + include_nested_fields: CatalogCommandOptions.include_nested_fields = None, + include_owners: CatalogCommandOptions.include_owners = False, + include_links: CatalogCommandOptions.include_links = False, + include_metadata: CatalogCommandOptions.include_metadata = False, + git_repositories: CatalogCommandOptions.git_repositories = None, + types: CatalogCommandOptions.types = None, + page: ListCommandOptions.page = None, + page_size: ListCommandOptions.page_size = 250, + table_output: ListCommandOptions.table_output = False, + csv_output: ListCommandOptions.csv_output = False, + columns: ListCommandOptions.columns = [], + no_headers: ListCommandOptions.no_headers = False, + filters: ListCommandOptions.filters = [], + sort: ListCommandOptions.sort = [], + _print: CommandOptions._print = True, +): + """ + List entities in the catalog """ client = ctx.obj["client"] + if (table_output or csv_output) and not ctx.params.get('columns'): + ctx.params['columns'] = [ + "ID=id", + "Tag=tag", + "Name=name", + "Type=type", + "Git Repository=git.repository", + ] + params = { - "dryRun":dry_run, - "appendArrays": append_arrays, - "deleteMarkerValue": delete_marker_value, - "failIfEntityDoesNotExist": fail_if_not_exist + "includeArchived": include_archived, + "hierarchyDepth": hierarchy_depth, + "groups": groups, + "owners": owners, + "includeHierarchyFields": include_hierarchy_fields, + "includeNestedFields": include_nested_fields, + "includeOwners": include_owners, + "includeLinks": include_links, + "includeMetadata": include_metadata, + "page": page, + "pageSize": page_size, + "gitRepositories": git_repositories, + "types": types, } - r = client.patch("api/v1/open-api", data=file_input.read(), params=params, content_type="application/openapi;charset=UTF-8") - print_output_with_context(ctx, r) + # remove any params that are None + params = {k: v for k, v in params.items() if v is not None} + + # for keys that can have multiple values, remove whitespace around comma and split on comma + for key in ['groups', 'owners', 'gitRepositories', 'types']: + if key in params: + params[key] = [x.strip() for x in params[key].split(',')] + + if page is None: + # if page is not specified, we want to fetch all pages + r = client.fetch("api/v1/catalog", params=params) + else: + # if page is specified, we want to fetch only that page + r = client.get("api/v1/catalog", params=params) + + if _print: + data = r + print_output_with_context(ctx, data) + else: + return(r) @app.command() def list_descriptors( @@ -339,16 +357,27 @@ def list_descriptors( return(r) @app.command() -def gitops_log( +def patch( ctx: typer.Context, - tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), + file_input: Annotated[typer.FileText, typer.Option(..., "--file", "-f", help=" File containing YAML content of entity; can be passed as stdin with -, example: -f-")] = None, + delete_marker_value = typer.Option("__delete__", "--delete-marker-value", "-dmv", help="Delete keys with this value from the merged yaml, defaults to __delete__, if any values match this, they will not be included in merged YAML. For example my_value: __delete__ will remove my_value from the merged YAML."), + dry_run: CatalogCommandOptions.dry_run = False, + append_arrays: CatalogCommandOptions.append_arrays = False, + fail_if_not_exist: CatalogCommandOptions.fail_if_not_exist = False, ): """ - Retrieve most recent GitOps log for entity + Creates or updates an entity. If the YAML refers to an entity that already exists (as referenced by the x-cortex-tag), this API will merge the specified changes into the existing entity """ client = ctx.obj["client"] - r = client.get("api/v1/catalog/" + tag + "/gitops-logs") + params = { + "dryRun":dry_run, + "appendArrays": append_arrays, + "deleteMarkerValue": delete_marker_value, + "failIfEntityDoesNotExist": fail_if_not_exist + } + + r = client.patch("api/v1/open-api", data=file_input.read(), params=params, content_type="application/openapi;charset=UTF-8") print_output_with_context(ctx, r) @app.command() @@ -363,3 +392,16 @@ def scorecard_scores( r = client.get("api/v1/catalog/" + tag + "/scorecards") print_output_with_context(ctx, r) + +@app.command() +def unarchive( + ctx: typer.Context, + tag: str = typer.Option(..., "--tag", "-t", help="The tag (x-cortex-tag) or unique, auto-generated identifier for the entity."), +): + """ + Unarchive an entity + """ + client = ctx.obj["client"] + + r = client.put("api/v1/catalog/" + tag + "/unarchive") + print_output_with_context(ctx, r) diff --git a/cortexapps_cli/commands/integrations_commands/aws.py b/cortexapps_cli/commands/integrations_commands/aws.py index 4f94a29..eab3b5b 100644 --- a/cortexapps_cli/commands/integrations_commands/aws.py +++ b/cortexapps_cli/commands/integrations_commands/aws.py @@ -146,7 +146,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/aws/configurations/validate" + accountId) + r = client.post("api/v1/aws/configurations/validate/" + accountId) print_json(data=r) @app.command() @@ -159,7 +159,7 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/aws/configurations") + r = client.post("api/v1/aws/configurations/all/validate") print_json(data=r) @app.command() diff --git a/cortexapps_cli/commands/integrations_commands/azure_devops.py b/cortexapps_cli/commands/integrations_commands/azure_devops.py index e1936fd..f641521 100644 --- a/cortexapps_cli/commands/integrations_commands/azure_devops.py +++ b/cortexapps_cli/commands/integrations_commands/azure_devops.py @@ -157,7 +157,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/azure-devops/configurations/validate" + alias) + r = client.post("api/v1/azure-devops/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -170,5 +170,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/azure-devops/configurations") + r = client.post("api/v1/azure-devops/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/azure_resources.py b/cortexapps_cli/commands/integrations_commands/azure_resources.py index 38743be..04bfdf1 100644 --- a/cortexapps_cli/commands/integrations_commands/azure_resources.py +++ b/cortexapps_cli/commands/integrations_commands/azure_resources.py @@ -183,7 +183,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/azure-resources/configurations/validate" + alias) + r = client.post("api/v1/azure-resources/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -196,7 +196,7 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/azure-resources/configurations") + r = client.post("api/v1/azure-resources/configuration/validate") print_json(data=r) @app.command() diff --git a/cortexapps_cli/commands/integrations_commands/circleci.py b/cortexapps_cli/commands/integrations_commands/circleci.py index 59ab7ed..b570a47 100644 --- a/cortexapps_cli/commands/integrations_commands/circleci.py +++ b/cortexapps_cli/commands/integrations_commands/circleci.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/circleci/configurations/validate" + alias) + r = client.post("api/v1/circleci/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/circleci/configurations") + r = client.post("api/v1/circleci/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/coralogix.py b/cortexapps_cli/commands/integrations_commands/coralogix.py index 3d42277..cf67c87 100644 --- a/cortexapps_cli/commands/integrations_commands/coralogix.py +++ b/cortexapps_cli/commands/integrations_commands/coralogix.py @@ -162,7 +162,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/coralogix/configurations/validate" + alias) + r = client.post("api/v1/coralogix/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -175,5 +175,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/coralogix/configurations") + r = client.post("api/v1/coralogix/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/datadog.py b/cortexapps_cli/commands/integrations_commands/datadog.py index d31e87a..6200aec 100644 --- a/cortexapps_cli/commands/integrations_commands/datadog.py +++ b/cortexapps_cli/commands/integrations_commands/datadog.py @@ -161,7 +161,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/datadog/configurations/validate" + alias) + r = client.post("api/v1/datadog/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -174,5 +174,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/datadog/configurations") + r = client.post("api/v1/datadog/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/gitlab.py b/cortexapps_cli/commands/integrations_commands/gitlab.py index 650a92b..c890f9b 100644 --- a/cortexapps_cli/commands/integrations_commands/gitlab.py +++ b/cortexapps_cli/commands/integrations_commands/gitlab.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/gitlab/configurations/validate" + alias) + r = client.post("api/v1/gitlab/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/gitlab/configurations") + r = client.post("api/v1/gitlab/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/incidentio.py b/cortexapps_cli/commands/integrations_commands/incidentio.py index 6faac76..d06d7ac 100644 --- a/cortexapps_cli/commands/integrations_commands/incidentio.py +++ b/cortexapps_cli/commands/integrations_commands/incidentio.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/incidentio/configurations/validate" + alias) + r = client.post("api/v1/incidentio/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/incidentio/configurations") + r = client.post("api/v1/incidentio/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/launchdarkly.py b/cortexapps_cli/commands/integrations_commands/launchdarkly.py index f3d9f03..c50556f 100644 --- a/cortexapps_cli/commands/integrations_commands/launchdarkly.py +++ b/cortexapps_cli/commands/integrations_commands/launchdarkly.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/launchdarkly/configurations/validate" + alias) + r = client.post("api/v1/launchdarkly/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/launchdarkly/configurations") + r = client.post("api/v1/launchdarkly/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/pagerduty.py b/cortexapps_cli/commands/integrations_commands/pagerduty.py index 5b38a92..7a5f17e 100644 --- a/cortexapps_cli/commands/integrations_commands/pagerduty.py +++ b/cortexapps_cli/commands/integrations_commands/pagerduty.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/pagerduty/configurations/validate" + alias) + r = client.post("api/v1/pagerduty/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/pagerduty/configurations") + r = client.post("api/v1/pagerduty/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/prometheus.py b/cortexapps_cli/commands/integrations_commands/prometheus.py index 2934d6e..0a3f2b6 100644 --- a/cortexapps_cli/commands/integrations_commands/prometheus.py +++ b/cortexapps_cli/commands/integrations_commands/prometheus.py @@ -159,7 +159,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/prometheus/configurations/validate" + alias) + r = client.post("api/v1/prometheus/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -172,5 +172,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/prometheus/configurations") + r = client.post("api/v1/prometheus/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/commands/integrations_commands/sonarqube.py b/cortexapps_cli/commands/integrations_commands/sonarqube.py index 0de0f3b..0c05715 100644 --- a/cortexapps_cli/commands/integrations_commands/sonarqube.py +++ b/cortexapps_cli/commands/integrations_commands/sonarqube.py @@ -153,7 +153,7 @@ def validate( client = ctx.obj["client"] - r = client.post("api/v1/sonarqube/configurations/validate" + alias) + r = client.post("api/v1/sonarqube/configuration/validate/" + alias) print_json(data=r) @app.command() @@ -166,5 +166,5 @@ def validate_all( client = ctx.obj["client"] - r = client.post("api/v1/sonarqube/configurations") + r = client.post("api/v1/sonarqube/configuration/validate") print_json(data=r) diff --git a/cortexapps_cli/cortex_client.py b/cortexapps_cli/cortex_client.py index 0a439ec..42b5e18 100644 --- a/cortexapps_cli/cortex_client.py +++ b/cortexapps_cli/cortex_client.py @@ -147,7 +147,12 @@ def request(self, method, endpoint, params={}, headers={}, data=None, raw_body=F self.rate_limiter.acquire() start_time = time.time() - response = self.session.request(method, url, params=params, headers=req_headers, data=req_data) + try: + response = self.session.request(method, url, params=params, headers=req_headers, data=req_data) + except requests.exceptions.ConnectionError as e: + print(f'[red][bold]Connection error[/bold][/red]: Could not connect to {url}') + print(f' [dim]{e}[/dim]') + raise typer.Exit(code=1) duration = time.time() - start_time # Log slow requests or non-200 responses (likely retries happened) diff --git a/internal/.env.default.example b/internal/.env.default.example new file mode 100644 index 0000000..08aa686 --- /dev/null +++ b/internal/.env.default.example @@ -0,0 +1,6 @@ +# Cloud workspace environment — copy to .env.default and fill in your values. +# Each team member's .env.default will differ (their own workspace + API key). +CORTEX_API_KEY= +CORTEX_BASE_URL=https://api.getcortexapp.com +CORTEX_APP_URL=https://app.getcortexapp.com +CORTEX_TENANT_CODE= diff --git a/internal/.env.local.example b/internal/.env.local.example new file mode 100644 index 0000000..afea16f --- /dev/null +++ b/internal/.env.local.example @@ -0,0 +1,6 @@ +# Local dev environment — Cortex running on host via bootRun. +# Copy to .env.local and fill in your API key from ~/.cortex/config [cortex-local]. +CORTEX_API_KEY= +CORTEX_BASE_URL=http://host.minikube.internal:8080 +CORTEX_APP_URL=http://app.local.getcortexapp.com:3000 +CORTEX_TENANT_CODE=cortex-local diff --git a/internal/CLAUDE.md b/internal/CLAUDE.md index 3d25428..4c71ea9 100644 --- a/internal/CLAUDE.md +++ b/internal/CLAUDE.md @@ -43,8 +43,24 @@ just axon-echo-setup # simple echo server relay smoke test ## Environment -- **`.env`** — local secrets and config (gitignored). Copy from `.env.example` and fill in values. +- **`.env`** — active config (gitignored). Copy from `.env.example` and fill in values. - **`set dotenv-load` + `set export`** in Justfile means all `.env` vars are auto-loaded and exported. + +### Environment profiles + +Switch between cloud and local dev with `just env`: + +```bash +just env # switch to cloud workspace (.env.default) +just env local # switch to local dev (.env.local) +just env-show # show active profile and key vars +``` + +Profile files (`.env.`) contain only the vars that differ per environment (API key, base URL, app URL, tenant code). `just env` patches those into `.env` — shared vars are untouched. + +Setup: +1. Copy `.env.default.example` → `.env.default` and fill in your cloud workspace values +2. Copy `.env.local.example` → `.env.local` and fill in your local API key (from `~/.cortex/config [cortex-local]`) - **`PYTHONPATH=..:../tests`** is set in pytest commands so internal tests can import both `cortexapps_cli` and `helpers.utils` from the parent project. ## Env var prompting diff --git a/internal/Justfile b/internal/Justfile index f0e3c4a..1a5854e 100644 --- a/internal/Justfile +++ b/internal/Justfile @@ -10,6 +10,25 @@ pw_pytest := 'poetry run pytest -rA --headed --browser chromium' help: @just -l +# --------------------------------------------------------------------------- +# Environment profiles +# --------------------------------------------------------------------------- + +# Switch environment profile (default=cloud workspace, local=local dev) +env profile="default": + @./scripts/switch-env.sh {{profile}} + +# Show the active environment profile +env-show: + @if [ -f .env.active ]; then \ + echo "Active profile: $(cat .env.active)"; \ + else \ + echo "No active profile (using .env as-is)"; \ + fi + @echo "" + @grep -E '^(CORTEX_API_KEY|CORTEX_BASE_URL|CORTEX_APP_URL|CORTEX_TENANT_CODE)=' .env 2>/dev/null | \ + sed 's/\(CORTEX_API_KEY=\).*/\1...redacted.../' || true + # --------------------------------------------------------------------------- # Shared helpers # --------------------------------------------------------------------------- @@ -104,21 +123,17 @@ k8s-agent-setup: --set image.tag="${IMAGE_TAG}" \ --set app.baseUrl="${CORTEX_BASE_URL}" - # 5. Wait for agent pod readiness + # 5. Restart agent pod to pick up any secret/configmap changes + kubectl rollout restart deployment -l app.kubernetes.io/name=cortex-k8s-agent + + # 6. Wait for agent pod readiness echo "Waiting for k8s-agent pod to be ready..." - for i in $(seq 1 30); do - if kubectl get pod -l app.kubernetes.io/name=cortex-k8s-agent 2>/dev/null | grep -q .; then - break - fi - sleep 2 - done - kubectl wait --for=condition=ready pod \ - -l app.kubernetes.io/name=cortex-k8s-agent \ - --timeout=120s + sleep 5 # Give k8s time to start the rollout + kubectl rollout status deployment -l app.kubernetes.io/name=cortex-k8s-agent --timeout=120s # 6. Create Cortex entity echo "Creating Cortex entity..." - {{cortex_cli}} catalog create -f k8s/cortex-entity.yaml || true + CORTEX_BASE_URL= CORTEX_API_KEY= {{cortex_cli}} -t "${CORTEX_TENANT_CODE}" catalog create -f k8s/cortex-entity.yaml || true # 7. Install Argo Rollouts CRD (required for Rollout test manifest) echo "Installing Argo Rollouts CRD..." @@ -128,6 +143,13 @@ k8s-agent-setup: echo "Applying test manifests..." kubectl apply -f k8s/manifests/ + # 9. Show initial agent logs + echo "" + echo "Agent logs (waiting 15s for first cache push)..." + sleep 15 + kubectl logs -l app.kubernetes.io/name=cortex-k8s-agent --tail=10 + + echo "" echo "Setup complete. Run 'just k8s-agent-test' to verify." # Verify test objects show up in Cortex (checks agent logs, then runs Playwright UI test) @@ -167,9 +189,41 @@ k8s-agent-test: exit 1 fi + # 3. Verify k8s data is accessible via the public API echo "" - echo "Agent is healthy and all test manifests are deployed." - echo "Run 'just test-k8s-agent-ui' to verify workloads appear in the Cortex UI." + echo "Checking k8s data via CLI..." + FAILED=0 + CLI_CMD="cortex -t ${CORTEX_TENANT_CODE} catalog k8s -t k8s-test-annotation" + echo " \$ ${CLI_CMD}" + K8S_RESULT=$(CORTEX_BASE_URL= CORTEX_API_KEY= {{cortex_cli}} -t "${CORTEX_TENANT_CODE}" catalog k8s -t k8s-test-annotation 2>&1) && RC=0 || RC=$? + if [ "$RC" -ne 0 ]; then + echo " FAILED: catalog k8s command exited with code $RC" + echo " $K8S_RESULT" | head -5 + FAILED=1 + elif echo "$K8S_RESULT" | grep -q '"resources"'; then + RESOURCE_COUNT=$(echo "$K8S_RESULT" | python3 -c "import sys,json; print(len(json.load(sys.stdin)['resources']))") + echo " OK: ${RESOURCE_COUNT} resource(s) returned" + echo "" + echo " Resources:" + echo "$K8S_RESULT" | python3 -c "import sys,json; [print(f' {r[\"type\"]:12s} {r[\"namespace\"]}/{r[\"name\"]} (cluster: {r[\"cluster\"]})') for r in json.load(sys.stdin).get('resources',[])]" + echo "" + echo " Sample JSON (first resource):" + echo "$K8S_RESULT" | python3 -c "import sys,json; r=json.load(sys.stdin)['resources'][0]; print(json.dumps({k:r[k] for k in ['type','namespace','name','cluster','lastUpdated']}, indent=4))" | sed 's/^/ /' + elif echo "$K8S_RESULT" | grep -q "404"; then + echo " FAILED: catalog k8s returned 404 — agent may not have pushed data yet." + FAILED=1 + else + echo " FAILED: unexpected response from catalog k8s:" + echo " $K8S_RESULT" | head -5 + FAILED=1 + fi + + echo "" + if [ "$FAILED" -eq 1 ] || [ "$MISSING" -eq 1 ]; then + echo "ERROR: One or more checks failed." + exit 1 + fi + echo "All checks passed." # Tear down k8s-agent (keeps minikube running) k8s-agent-stop: diff --git a/internal/k8s/cortex-entity.yaml b/internal/k8s/cortex-entity.yaml index baba670..41a5cd1 100644 --- a/internal/k8s/cortex-entity.yaml +++ b/internal/k8s/cortex-entity.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: K8s Test Service + title: K8s Test Annotation description: Test entity for K8s agent integration x-cortex-tag: k8s-test-annotation x-cortex-type: service diff --git a/internal/k8s/helm-chart/.helmignore b/internal/k8s/helm-chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/internal/k8s/helm-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/internal/k8s/helm-chart/README.md b/internal/k8s/helm-chart/README.md new file mode 100644 index 0000000..a6dfa9a --- /dev/null +++ b/internal/k8s/helm-chart/README.md @@ -0,0 +1,28 @@ +# Cortex k8s Helm Chart + +## Requirements +* [Helm](https://helm.sh/docs/intro/install/) +* A token for our package registry + +## Process +1. Generate a new Cortex API Key on the [API Keys Settings tab](https://app.getcortexapp.com/admin/settings/api-keys) in Cortex. + - This will be used for the Cortex Kubernetes agent to communicate and push service information to Cortex backend without exposing your public API Key. +2. Inside your Kubernetes cluster, run the following command to generate a Kubernetes secret for the Cortex API Key. + `kubectl create secret generic cortex-key --from-literal api-key=YOUR_API_KEY` +3. Run `kubectl create secret docker-registry cortex-docker-registry-secret --docker-server=ghcr.io --docker-username=$GITHUB_USERNAME --docker-password=$GITHUB_PASSWORD --docker-email=` +4. Download the helm chart and inside the repository run the following command to install the agent in your cluster. + `helm install YOUR_SELECTED_CHART_NAME .` + +## Customization +The helm chart make installation quick and simple, but if you want to customize any of the installation features for the Cortex agent you can do so by changing the following information in the `values.yaml` of the helm chart. +### Service Account +To authenticate the Cortex agent in your cluster and grant it access to service information, the agent needs its own service account. The helm chart by default creates a Service Account `cortex-service-account`, but you can customize the `name` and `namespace` of this Service Account. If you already have a Service Account that you want the Cortex agent to use, set `create: false` under `serviceAccount` and enter the `name` and `namespace` of the Service Account you wish to use. +### Service +The service type and port can be customized as well. For security, the agent uses a default `ClusterIP` service type that only allows the service to be accessed from within the cluster. +### Resources +By default, no resources are specified. While the Cortex Kubernetes agent is designed to be lightweight and minimize resource utilization, you have the option to add custom CPU limits and requests. +### Base URL +The Base URL defaults to that for the hosted version of Cortex. If you are using the on-prem version of Cortex, you should change the `app/baseUrl` value to the correct URL for your on-prem Cortex. + +# Usage +After installation, usage is very simple as no additional steps are required. The next time you go to create a new service in your Service Directory Homepage, you should see all of your Kubernetes services already added, ready for you to use in Cortex. If you do not want to import all of your Kubernetes discovered services, you can simply remove the ones you do not want to add. Removed services will still show up in the Kubernetes tab of Discovered Services if you want to go back and add them later. diff --git a/internal/k8s/helm-chart/templates/deployment.yaml b/internal/k8s/helm-chart/templates/deployment.yaml index 3966201..fb3deb2 100644 --- a/internal/k8s/helm-chart/templates/deployment.yaml +++ b/internal/k8s/helm-chart/templates/deployment.yaml @@ -11,8 +11,9 @@ spec: {{- include "helm-chart.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: @@ -22,6 +23,12 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + ######### remove before deploy - used for local testing ########### + # hostAliases: + # - ip: "192.168.64.1" + # hostnames: + # - "host.minikube.internal" + ################################################################### serviceAccountName: {{ include "helm-chart.serviceAccountName" . }} containers: - name: {{ .Chart.Name }} diff --git a/internal/k8s/manifests/argo-deploy-rollout.yaml b/internal/k8s/manifests/argo-deploy-rollout.yaml new file mode 100644 index 0000000..9f61f34 --- /dev/null +++ b/internal/k8s/manifests/argo-deploy-rollout.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: argo-deploy + labels: + app: k8s-test-label + annotations: + cortex.io/tag: k8s-test-annotation +spec: + replicas: 1 + selector: + matchLabels: + app: argo-deploy + template: + metadata: + labels: + app: argo-deploy + spec: + containers: + - name: hello + image: nginx:alpine + ports: + - containerPort: 80 + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "64Mi" + cpu: "50m" + strategy: + canary: + steps: + - setWeight: 20 + - pause: {duration: 5m} + - setWeight: 50 + - pause: {duration: 5m} + - setWeight: 100 diff --git a/internal/k8s/manifests/argo-workloadref-rollout.yaml b/internal/k8s/manifests/argo-workloadref-rollout.yaml new file mode 100644 index 0000000..913b136 --- /dev/null +++ b/internal/k8s/manifests/argo-workloadref-rollout.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: argo-workloadref-deploy + labels: + app: k8s-test-label + annotations: + cortex.io/tag: k8s-test-annotation +spec: + replicas: 0 + selector: + matchLabels: + app: argo-workloadref + template: + metadata: + labels: + app: argo-workloadref + spec: + containers: + - name: hello + image: nginx:alpine + ports: + - containerPort: 80 + resources: + requests: + memory: "32Mi" + cpu: "10m" + limits: + memory: "64Mi" + cpu: "50m" +--- +apiVersion: argoproj.io/v1alpha1 +kind: Rollout +metadata: + name: argo-workloadref + labels: + app: k8s-test-label + annotations: + cortex.io/tag: k8s-test-annotation +spec: + replicas: 1 + selector: + matchLabels: + app: argo-workloadref + workloadRef: + apiVersion: apps/v1 + kind: Deployment + name: argo-workloadref-deploy + strategy: + canary: + steps: + - setWeight: 20 + - pause: {duration: 5m} + - setWeight: 50 + - pause: {duration: 5m} + - setWeight: 100 diff --git a/internal/scripts/switch-env.sh b/internal/scripts/switch-env.sh new file mode 100755 index 0000000..6b02b23 --- /dev/null +++ b/internal/scripts/switch-env.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Switch environment profile by patching .env with values from .env.. +# +# Profile files contain only the vars that differ per environment (e.g., +# CORTEX_API_KEY, CORTEX_BASE_URL). All other vars in .env are left untouched. +# +# Usage: +# ./scripts/switch-env.sh local # switch to .env.local +# ./scripts/switch-env.sh # switch to .env.default (cloud workspace) +set -euo pipefail + +PROFILE="${1:-default}" +PROFILE_FILE=".env.${PROFILE}" + +if [ ! -f "$PROFILE_FILE" ]; then + echo "Profile not found: $PROFILE_FILE" + echo "" + echo "Available profiles:" + for f in .env.*; do + # Skip .env.example and .env.active + case "$f" in + .env.example|.env.active) continue ;; + .env.*) echo " ${f#.env.}" ;; + esac + done + exit 1 +fi + +# Create .env from .env.example if it doesn't exist +if [ ! -f .env ]; then + if [ -f .env.example ]; then + cp .env.example .env + echo "Created .env from .env.example" + else + touch .env + fi +fi + +# Patch: for each KEY=VALUE in the profile, update or append in .env +while IFS= read -r line || [ -n "$line" ]; do + # Skip comments and blank lines + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line// /}" ]] && continue + + KEY=$(echo "$line" | cut -d'=' -f1) + # Remove existing line for this key (if any), then append the new one + grep -v "^${KEY}=" .env > .env.tmp || true + mv .env.tmp .env + echo "$line" >> .env +done < "$PROFILE_FILE" + +# Record active profile +echo "$PROFILE" > .env.active + +echo "Switched to profile: $PROFILE" +echo "" +grep -E '^(CORTEX_API_KEY|CORTEX_BASE_URL|CORTEX_APP_URL|CORTEX_TENANT_CODE)=' .env | \ + sed 's/\(CORTEX_API_KEY=\).*/\1...redacted.../' diff --git a/tests/test_catalog_k8s.py b/tests/test_catalog_k8s.py new file mode 100644 index 0000000..bdbf8e4 --- /dev/null +++ b/tests/test_catalog_k8s.py @@ -0,0 +1,86 @@ +from tests.helpers.utils import * + +BASE_URL = "https://api.getcortexapp.com" + +MOCK_K8S_RESPONSE = { + "resources": [ + { + "namespace": "production", + "name": "my-service", + "cluster": "prod-cluster", + "type": "Deployment", + "lastUpdated": "2024-01-15T10:30:00Z", + }, + { + "namespace": "production", + "name": "my-service-worker", + "cluster": "prod-cluster", + "type": "StatefulSet", + "lastUpdated": "2024-01-15T10:30:00Z", + }, + ] +} + +TAG = "my-service" + + +@responses.activate +def test_catalog_k8s_json(): + responses.add( + responses.GET, + BASE_URL + f"/api/v1/catalog/{TAG}/k8s", + json=MOCK_K8S_RESPONSE, + status=200, + ) + response = cli(["catalog", "k8s", "--tag", TAG]) + assert response == MOCK_K8S_RESPONSE + + +@responses.activate +def test_catalog_k8s_table(): + responses.add( + responses.GET, + BASE_URL + f"/api/v1/catalog/{TAG}/k8s", + json=MOCK_K8S_RESPONSE, + status=200, + ) + response = cli(["catalog", "k8s", "--tag", TAG, "--table"], ReturnType.STDOUT) + assert "production" in response + assert "prod-cluster" in response + + +@responses.activate +def test_catalog_k8s_csv(): + responses.add( + responses.GET, + BASE_URL + f"/api/v1/catalog/{TAG}/k8s", + json=MOCK_K8S_RESPONSE, + status=200, + ) + response = cli(["catalog", "k8s", "--tag", TAG, "--csv"], ReturnType.STDOUT) + assert "production" in response + assert "prod-cluster" in response + + +@responses.activate +def test_catalog_k8s_table_and_csv_raises_error(): + responses.add( + responses.GET, + BASE_URL + f"/api/v1/catalog/{TAG}/k8s", + json=MOCK_K8S_RESPONSE, + status=200, + ) + result = cli(["catalog", "k8s", "--tag", TAG, "--table", "--csv"], ReturnType.RAW) + assert result.exit_code != 0 + + +@responses.activate +def test_catalog_k8s_empty_response(): + responses.add( + responses.GET, + BASE_URL + f"/api/v1/catalog/{TAG}/k8s", + json={}, + status=200, + ) + response = cli(["catalog", "k8s", "--tag", TAG]) + assert response == {} diff --git a/tests/test_config_file.py b/tests/test_config_file.py index 1f86c97..b2962c9 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -53,7 +53,7 @@ def test_config_file_bad_url(monkeypatch, tmp_path): content = template.substitute(cortex_api_key=cortex_api_key) f.write_text(content) response = cli(["-c", str(f), "-l", "DEBUG", "-t", "mySection", "entity-types", "list"], return_type=ReturnType.RAW) - assert "Max retries exceeded with url" in str(response), "should get max retries error" + assert "Connection error" in response.output, "should get connection error" def test_config_file_base_url_env_var(monkeypatch, tmp_path): cortex_api_key = os.getenv('CORTEX_API_KEY') diff --git a/tests/test_integrations_azure_resources.py b/tests/test_integrations_azure_resources.py index 49f2576..e736ecd 100644 --- a/tests/test_integrations_azure_resources.py +++ b/tests/test_integrations_azure_resources.py @@ -60,10 +60,10 @@ def test_integrations_azure_resources_validate_all(): @responses.activate def test_integrations_list_types(): - responses.add(responses.GET, os.getenv("CORTEX_BASE_URL") + "/api/v1/azure-resoures/types", json={}, status=200) + responses.add(responses.GET, os.getenv("CORTEX_BASE_URL") + "/api/v1/azure-resources/types", json={}, status=200) cli(["integrations", "azure-resources", "list-types"]) @responses.activate def test_integrations_azure_resoures_update_types(): - responses.add(responses.PUT, os.getenv("CORTEX_BASE_URL") + "/api/v1/azure-resoures/types", json={}, status=200) + responses.add(responses.PUT, os.getenv("CORTEX_BASE_URL") + "/api/v1/azure-resources/types", json={}, status=200) cli(["integrations", "azure-resources", "update-types", "-t", "microsoft.insights/workbooks=true", "-t", "microsoft.resources/subscriptions=false"], ReturnType.RAW) diff --git a/tests/test_integrations_sonarqube.py b/tests/test_integrations_sonarqube.py index dafd6d4..fc45855 100644 --- a/tests/test_integrations_sonarqube.py +++ b/tests/test_integrations_sonarqube.py @@ -36,7 +36,7 @@ def test_integrations_sonarqube_add(): @responses.activate def test_integrations_sonarqube_add_multiple(tmp_path): f = _dummy_file(tmp_path) - responses.add(responses.POST, os.getenv("CORTEX_BASE_URL") + "/api/v1/sonarqube/configurations", json={}, status=200) + responses.add(responses.PUT, os.getenv("CORTEX_BASE_URL") + "/api/v1/sonarqube/configurations", json={}, status=200) cli(["integrations", "sonarqube", "add-multiple", "-f", str(f)]) @responses.activate