Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ linters:
- common-false-positives
- legacy
- std-error-handling
rules:
- linters:
- goconst
path: _test\.go$
paths:
- third_party$
- builtin$
Expand Down
23 changes: 17 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -458,21 +458,32 @@ This project uses GitHub Actions and GoReleaser for automated releases:
### Dependency updates

Renovate runs daily and keeps compiler updates separate from library updates.
Routine updates wait until releases are at least seven days old.
Known vulnerability fixes and build-environment patches have no release delay.
OSV alerts identify known vulnerable dependencies and trigger fix PRs.
GitHub Actions patches, minors, and digest updates use a separate automerge PR.
Major action upgrades require manual review.
CI tools use a separate manual group; their major upgrades get individual PRs.
Module patches and `golang.org/x/*` module minors share one library PR.
Manual AWS SDK and Kubernetes client updates have separate groups.
Other manual module updates get individual PRs.
GoReleaser patches share the compiler-patch group and use native automerge.
GoReleaser minors remain in the manual CI tool group.
Module patches and minors use three groups: AWS SDK, Kubernetes, and other Go libraries.
Each group keeps patches separate from minors.
The other-library group includes ClickHouse, Elasticsearch, OpenTelemetry,
OpenAPI, and `golang.org/x/*`.
Module patches, AWS SDK minors, and other-library minors use native automerge
after artifact validation and App approval.
Kubernetes minors stay grouped for manual review.
Kubernetes upgrades select compatible `kube-openapi` revisions instead of independent Renovate updates.
Renovate allows ten concurrent PRs and inherits the shared hourly creation limit.
Compiler updates stay within the release line declared in `renovate.json`.
Module candidates that declare a newer required Go release line are filtered out.
CI rejects a module update that selects a compiler from another release line.
Manual groups rebase only when conflicting; automatic groups stay current with main.
The shared approval action approves verified Renovate commits with native automerge enabled.
GitHub requires the existing CI checks and an approving review before merging.
Compiler patches and the library group use native automerge after artifact validation and App approval.
The library group includes direct and indirect modules.
Other module minors, module majors, and Go release-line migrations require manual review.
Compiler patches use native automerge after artifact validation and App approval.
Library updates include direct and indirect modules.
Module majors and Go release-line migrations require individual manual review.
Dependency updates reach published binaries through the existing tag release process.

The approval workflow calls the pinned public `renovate-config` action directly.
Expand Down
2 changes: 1 addition & 1 deletion cmd/settings/check_and_finalize.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func runCheckAndFinalize(appCtx *app.Context) error {
K8sClient: appCtx.K8sClient,
Namespace: appCtx.Namespace,
JobName: checkJobName,
ServiceName: "settings",
ServiceName: settingsServiceName,
Comment thread
rb3ckers marked this conversation as resolved.
ScaleUpFn: scale.ScaleUpAndReleaseLock,
ScaleDownFn: scale.ScaleDown,
ScaleSelector: appCtx.Config.Settings.Restore.ScaleDownLabelSelector,
Expand Down
2 changes: 1 addition & 1 deletion cmd/settings/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ func buildVolumes(config *config.Config, defaultMode int32) []corev1.Volume {
// buildContainers constructs containers for the restore job
func buildContainer(envVar []corev1.EnvVar, command []string, config *config.Config) corev1.Container {
return corev1.Container{
Name: "settings",
Name: settingsServiceName,
Image: config.Settings.Restore.Job.Image,
ImagePullPolicy: corev1.PullIfNotPresent,
SecurityContext: k8s.ConvertSecurityContext(config.Settings.Restore.Job.ContainerSecurityContext),
Expand Down
4 changes: 3 additions & 1 deletion cmd/settings/settings.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import (
"github.com/stackvista/stackstate-backup-cli/internal/foundation/config"
)

const settingsServiceName = "settings"

func Cmd(globalFlags *config.CLIGlobalFlags) *cobra.Command {
cmd := &cobra.Command{
Use: "settings",
Use: settingsServiceName,
Short: "Settings backup and restore operations",
}

Expand Down
15 changes: 9 additions & 6 deletions cmd/stackgraphv2/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ const (
restoreNameTemplate = "stackgraph-v2-restore"
configMapDefaultFileMode = 0755
purgeStackgraphDataFlag = "-force"
backupLogVolumeName = "backup-log"
restoreScriptsVolumeName = "backup-restore-scripts"
storageKeysVolumeName = "minio-keys"
)

// Restore command flags
Expand Down Expand Up @@ -306,9 +309,9 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV
// buildRestoreVolumeMounts constructs volume mounts for the restore job container
func buildRestoreVolumeMounts(config *config.Config) []corev1.VolumeMount {
volumeMounts := []corev1.VolumeMount{
{Name: "backup-log", MountPath: "/opt/docker/etc_log"},
{Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"},
{Name: "minio-keys", MountPath: "/aws-keys"},
{Name: backupLogVolumeName, MountPath: "/opt/docker/etc_log"},
{Name: restoreScriptsVolumeName, MountPath: "/backup-restore-scripts"},
{Name: storageKeysVolumeName, MountPath: "/aws-keys"},
{Name: "tmp-data", MountPath: "/tmp-data"},
}

Expand Down Expand Up @@ -344,7 +347,7 @@ func buildRestoreInitContainers(config *config.Config) []corev1.Container {
func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int32) []corev1.Volume {
volumes := []corev1.Volume{
{
Name: "backup-log",
Name: backupLogVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand All @@ -354,7 +357,7 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3
},
},
{
Name: "backup-restore-scripts",
Name: restoreScriptsVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand All @@ -365,7 +368,7 @@ func buildRestoreVolumes(jobName string, config *config.Config, defaultMode int3
},
},
{
Name: "minio-keys",
Name: storageKeysVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: restore.MinioKeysSecretName,
Expand Down
12 changes: 6 additions & 6 deletions cmd/stackgraphv2/simplejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ func buildSimpleJobEnvVars(config *config.Config) []corev1.EnvVar {
// buildSimpleJobVolumeMounts constructs volume mounts shared by the abort and backfill job containers
func buildSimpleJobVolumeMounts() []corev1.VolumeMount {
return []corev1.VolumeMount{
{Name: "backup-log", MountPath: "/opt/docker/etc_log"},
{Name: "backup-restore-scripts", MountPath: "/backup-restore-scripts"},
{Name: "minio-keys", MountPath: "/aws-keys"},
{Name: backupLogVolumeName, MountPath: "/opt/docker/etc_log"},
{Name: restoreScriptsVolumeName, MountPath: "/backup-restore-scripts"},
{Name: storageKeysVolumeName, MountPath: "/aws-keys"},
}
}

Expand All @@ -86,7 +86,7 @@ func buildSimpleJobInitContainers(config *config.Config) []corev1.Container {
func buildSimpleJobVolumes(config *config.Config, defaultMode int32) []corev1.Volume {
return []corev1.Volume{
{
Name: "backup-log",
Name: backupLogVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand All @@ -96,7 +96,7 @@ func buildSimpleJobVolumes(config *config.Config, defaultMode int32) []corev1.Vo
},
},
{
Name: "backup-restore-scripts",
Name: restoreScriptsVolumeName,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Expand All @@ -107,7 +107,7 @@ func buildSimpleJobVolumes(config *config.Config, defaultMode int32) []corev1.Vo
},
},
{
Name: "minio-keys",
Name: storageKeysVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: restore.MinioKeysSecretName,
Expand Down
15 changes: 9 additions & 6 deletions internal/orchestration/replication/applicability.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ import (
)

const (
hdfsComponent = "hdfs"
clickhouseComponent = "clickhouse"
nameNodeComponent = "hdfs-nn"
dataNodeComponent = "hdfs-dn"
monoComponent = "stackgraph"
hdfsComponent = "hdfs"
clickhouseComponent = "clickhouse"
elasticsearchComponent = "elasticsearch"
kafkaComponent = "kafka"
zookeeperComponent = "zookeeper"
nameNodeComponent = "hdfs-nn"
dataNodeComponent = "hdfs-dn"
monoComponent = "stackgraph"
)

func workloadComponent(workload appsv1.StatefulSet) string {
Expand All @@ -26,7 +29,7 @@ func workloadComponent(workload appsv1.StatefulSet) string {
}
}
switch name {
case "elasticsearch", clickhouseComponent, "kafka", "zookeeper":
case elasticsearchComponent, clickhouseComponent, kafkaComponent, zookeeperComponent:
return name
default:
return ""
Expand Down
17 changes: 11 additions & 6 deletions internal/orchestration/replication/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ import (
"time"
)

const requestTimeout = 30 * time.Second
const (
requestTimeout = 30 * time.Second
bashExecutable = "bash"
bashExecFlag = "-ec"
replicationScriptName = "replication-check"
)

// Checker observes databases without invoking backup or restore operations.
type Checker struct {
Expand Down Expand Up @@ -83,15 +88,15 @@ func (c *Checker) query(ctx context.Context, pod, container string, command []st

func (c *Checker) checkComponent(ctx context.Context, inventory inventory, component string) Result {
switch component {
case "hdfs":
case hdfsComponent:
return c.checkHDFS(ctx, inventory)
case "elasticsearch":
case elasticsearchComponent:
return c.checkElasticsearch(ctx, inventory)
case "kafka":
case kafkaComponent:
return c.checkKafka(ctx, inventory)
case "clickhouse":
case clickhouseComponent:
return c.checkClickHouse(ctx, inventory)
case "zookeeper":
case zookeeperComponent:
return c.checkZooKeeper(ctx, inventory)
default:
return result(component, Unknown, "unsupported component")
Expand Down
6 changes: 3 additions & 3 deletions internal/orchestration/replication/clickhouse.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (c *Checker) checkClickHouse(ctx context.Context, inventory inventory) Resu
if member.replicas < minReplicas {
return result("clickhouse", Degraded, fmt.Sprintf("%s belongs to a shard with fewer than two replicas", member.pod.Name))
}
command := []string{"bash", "-ec", clickhouseQuery, "replication-check", clickhouseSQL}
command := []string{bashExecutable, bashExecFlag, clickhouseQuery, replicationScriptName, clickhouseSQL}
data, err := c.query(ctx, member.pod.Name, "clickhouse", command)
if err != nil {
return result("clickhouse", Unknown, err.Error())
Expand All @@ -67,7 +67,7 @@ func (c *Checker) verifyEmptyLogs(ctx context.Context, pod string, rows []replic
if !rows[n].needsLogVerification {
continue
}
command := []string{"bash", "-ec", clickhouseQuery, "replication-check", clickhouseEmptyLogSQL, "--param_log_path=" + rows[n].path + "/log"}
command := []string{bashExecutable, bashExecFlag, clickhouseQuery, replicationScriptName, clickhouseEmptyLogSQL, "--param_log_path=" + rows[n].path + "/log"}
data, err := c.query(ctx, pod, "clickhouse", command)
if err != nil {
return fmt.Errorf("%s: could not verify empty replication log: %w", pod, err)
Expand Down Expand Up @@ -178,7 +178,7 @@ func evaluateClickHouse(observations []replicaObservation) Result {
}
if len(problems) > 0 {
sort.Strings(problems)
return Result{Component: "clickhouse", Status: Degraded, Messages: problems}
return Result{Component: clickhouseComponent, Status: Degraded, Messages: problems}
}
report := result("clickhouse", Healthy, fmt.Sprintf("%d replicated table groups checked on every member; no pending data replication tasks", len(groups)))
if historicalErrors > 0 {
Expand Down
4 changes: 2 additions & 2 deletions internal/orchestration/replication/elasticsearch.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func (c *Checker) checkElasticsearch(ctx context.Context, inventory inventory) R
if err := expectedMembers(members); err != nil {
return result("elasticsearch", Degraded, err.Error())
}
command := []string{"bash", "-ec", elasticsearchQuery}
command := []string{bashExecutable, bashExecFlag, elasticsearchQuery}
data, err := c.query(ctx, members[0].pod.Name, "elasticsearch", command)
if err != nil {
return result("elasticsearch", Unknown, err.Error())
Expand Down Expand Up @@ -66,7 +66,7 @@ func evaluateElasticsearch(data []byte, expected int) Result {
}
if len(problems) > 0 {
sort.Strings(problems)
return Result{Component: "elasticsearch", Status: Degraded, Messages: problems}
return Result{Component: elasticsearchComponent, Status: Degraded, Messages: problems}
}
return result("elasticsearch", Healthy, fmt.Sprintf("%d indices have replica shards; all shards allocated with no recovery or relocation", len(indices)))
}
4 changes: 2 additions & 2 deletions internal/orchestration/replication/hdfs.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (c *Checker) checkHDFS(ctx context.Context, inventory inventory) Result {
if err := expectedMembers(datanodes); err != nil {
return result("hdfs", Degraded, err.Error())
}
data, err := c.query(ctx, namenodes[0].pod.Name, "namenode", []string{"bash", "-ec", hdfsQuery})
data, err := c.query(ctx, namenodes[0].pod.Name, "namenode", []string{bashExecutable, bashExecFlag, hdfsQuery})
if err != nil {
return result("hdfs", Unknown, err.Error())
}
Expand Down Expand Up @@ -85,7 +85,7 @@ func evaluateHDFS(data []byte, expected int) Result {
}
}
if len(problems) > 0 {
return Result{Component: "hdfs", Status: Degraded, Messages: problems}
return Result{Component: hdfsComponent, Status: Degraded, Messages: problems}
}
return result("hdfs", Healthy, fmt.Sprintf("%d DataNodes live; no missing, corrupt, under-replicated or pending-replication blocks", expected))
}
2 changes: 1 addition & 1 deletion internal/orchestration/replication/hdfs_audit.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (c *Checker) Verify(ctx context.Context, previous Report) Report {

func (c *Checker) auditHDFS(ctx context.Context, pod string) Result {
parser := &fsckParser{}
err := c.kube.ExecTo(ctx, c.options.Namespace, pod, "namenode", []string{"bash", "-ec", hdfsAuditQuery}, parser)
err := c.kube.ExecTo(ctx, c.options.Namespace, pod, "namenode", []string{bashExecutable, bashExecFlag, hdfsAuditQuery}, parser)
if ctx.Err() != nil {
return result(hdfsComponent, Unknown, fmt.Sprintf("HDFS block audit did not complete: %v", ctx.Err()))
}
Expand Down
2 changes: 1 addition & 1 deletion internal/orchestration/replication/hdfs_fsck.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (p *fsckParser) result() Result {
return result("hdfs", Unknown, "incomplete fsck report; block replication could not be verified")
}
if p.problems > 0 {
return Result{Component: "hdfs", Status: Degraded,
return Result{Component: hdfsComponent, Status: Degraded,
Messages: append([]string{fmt.Sprintf("HDFS block audit found %d replication problems", p.problems)}, p.messages...)}
}
report := result("hdfs", Healthy, fmt.Sprintf("HDFS audit: %d files and %d completed block entries meet replication targets of at least two (including snapshot references)", p.files, p.completed))
Expand Down
6 changes: 3 additions & 3 deletions internal/orchestration/replication/kafka.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func (c *Checker) checkKafka(ctx context.Context, inventory inventory) Result {
if err := expectedMembers(members); err != nil {
return result("kafka", Degraded, err.Error())
}
command := []string{"bash", "-ec", kafkaQuery, "replication-check", "--bootstrap-server", "localhost:9092", "--describe"}
command := []string{bashExecutable, bashExecFlag, kafkaQuery, replicationScriptName, "--bootstrap-server", "localhost:9092", "--describe"}
data, err := c.query(ctx, members[0].pod.Name, "kafka", command)
if err != nil {
return result("kafka", Unknown, err.Error())
Expand All @@ -48,7 +48,7 @@ func (c *Checker) checkKafka(ctx context.Context, inventory inventory) Result {
if report.Status == Unknown || hasTransactionTopic(string(data)) {
return report
}
probe := []string{"bash", "-ec", transactionTopicQuery, "replication-check", "--bootstrap-server", "localhost:9092"}
probe := []string{bashExecutable, bashExecFlag, transactionTopicQuery, replicationScriptName, "--bootstrap-server", "localhost:9092"}
presence, err := c.query(ctx, members[0].pod.Name, "kafka", probe)
if err != nil || strings.TrimSpace(string(presence)) != "absent" {
report.Status = Unknown
Expand Down Expand Up @@ -144,7 +144,7 @@ func evaluateKafka(output string) Result {
}
report := result("kafka", Healthy, fmt.Sprintf("all partitions of %d topics have at least two replicas, complete ISR and an in-sync leader", len(counts)))
if len(problems) > 0 {
report = Result{Component: "kafka", Status: Degraded, Messages: problems}
report = Result{Component: kafkaComponent, Status: Degraded, Messages: problems}
}
return report
}
Expand Down
2 changes: 1 addition & 1 deletion internal/orchestration/replication/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const (
minReplicas = 2
)

var supportedComponents = []string{"hdfs", "elasticsearch", "kafka", "clickhouse", "zookeeper"}
var supportedComponents = []string{hdfsComponent, elasticsearchComponent, kafkaComponent, clickhouseComponent, zookeeperComponent}

// Options identifies the installation and selected databases.
type Options struct {
Expand Down
4 changes: 2 additions & 2 deletions internal/orchestration/replication/zookeeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func (c *Checker) checkZooKeeper(ctx context.Context, inventory inventory) Resul
}

func (c *Checker) queryZooKeeper(ctx context.Context, pod string) (zooKeeperObservation, error) {
data, err := c.query(ctx, pod, "zookeeper", []string{"bash", "-ec", zookeeperQuery})
data, err := c.query(ctx, pod, "zookeeper", []string{bashExecutable, bashExecFlag, zookeeperQuery})
if err != nil {
return zooKeeperObservation{}, err
}
Expand Down Expand Up @@ -163,7 +163,7 @@ func evaluateZooKeeper(observations []zooKeeperObservation) Result {
problems = append(problems, fmt.Sprintf("expected one ZooKeeper leader; observed %d", leaders))
}
if len(problems) > 0 {
return Result{Component: "zookeeper", Status: Degraded, Messages: problems}
return Result{Component: zookeeperComponent, Status: Degraded, Messages: problems}
}
return result("zookeeper", Healthy, fmt.Sprintf("%d voting members available; one leader and %d synchronized followers", len(observations), len(observations)-1))
}
Loading
Loading