Apl 1476

chart/apl/templates/deployment.yaml

@@ -75,13 +75,9 @@ spec:
               value: {{ .Values.operator.pollIntervalMs | default "30000" | quote }}
             - name: RECONCILE_INTERVAL_MS
               value: {{ .Values.operator.reconcileIntervalMs | default "300000" | quote }}
-          {{- if hasKey $kms "sops" }}
           envFrom:
-            - secretRef:
-                name: apl-sops-secrets
             - secretRef:
                 name: gitea-credentials
-          {{- end }}
           volumeMounts:
             - name: otomi-values
               mountPath: /home/app/stack/env

charts/ingress-nginx/templates/clusterrole.yaml

@@ -27,6 +27,7 @@ rules:
       - namespaces
 {{- end}}
     verbs:
+      - get
       - list
       - watch
   - apiGroups:

helmfile.d/helmfile-03.databases.yaml.gotmpl

@@ -14,13 +14,6 @@ bases:
 {{- $k := $a.keycloak }}
 
 releases:
-  - name: gitea-db-secret-artifacts
-    installed: true
-    namespace: gitea
-    labels:
-      pkg: gitea
-      app: core
-    <<: *raw
   - name: gitea-otomi-db
     installed: true
     namespace: gitea

helmfile.d/snippets/sops-env.gotmpl

@@ -1,24 +0,0 @@
-{{- with . | get "azure" nil }}
-AZURE_CLIENT_ID: {{ .clientId }}
-AZURE_CLIENT_SECRET: {{ .clientSecret }}
-{{- with . | get "tenantId" nil }}
-AZURE_TENANT_ID: {{ . }}{{ end }}
-{{- with . | get "environment" nil }}
-AZURE_ENVIRONMENT: {{ . }}{{ end }}
-{{- end }}
-{{- with . | get "aws" nil }}
-AWS_ACCESS_KEY_ID: {{ .accessKey }}
-AWS_SECRET_ACCESS_KEY: {{ .secretKey }}
-{{- with . | get "region" nil }}
-AWS_REGION: {{ . }}{{ end }}
-{{- end }}
-{{- with . | get "age" nil }}
-SOPS_AGE_KEY: {{ .privateKey }}
-{{- end }}
-{{- with . | get "google" nil }}
-GCLOUD_SERVICE_KEY: '{{ .accountJson | replace "\n" "" }}'
-{{- with . | get "project" nil }}
-GOOGLE_PROJECT: {{ . }}{{ end }}
-{{- with . | get "region" nil }}
-GOOGLE_REGION: {{ . }}{{ end }}
-{{- end }}

src/cmd/bootstrap.test.ts

@@ -12,6 +12,8 @@ import {
   processValues,
 } from './bootstrap'
 
+jest.mock('@linode/kubeseal-encrypt')
+
 const { terminal } = stubs
 
 jest.mock('src/common/envalid', () => ({
@@ -37,6 +39,7 @@ describe('Bootstrapping values', () => {
         }),
       }),
       bootstrapSops: jest.fn(),
+      bootstrapSealedSecrets: jest.fn(),
       copyBasicFiles: jest.fn(),
       copyFile: jest.fn(),
       createCustomCA: jest.fn(),
@@ -59,14 +62,14 @@ describe('Bootstrapping values', () => {
     }
   })
   it('should call relevant sub routines', async () => {
-    deps.processValues.mockReturnValue(values)
+    deps.processValues.mockReturnValue({ originalInput: values, allSecrets: {} })
     deps.hfValues.mockReturnValue(values)
     await bootstrap(deps)
     expect(deps.copyBasicFiles).toHaveBeenCalled()
-    expect(deps.bootstrapSops).toHaveBeenCalled()
+    expect(deps.bootstrapSealedSecrets).toHaveBeenCalled()
   })
   it('should copy only skeleton files to env dir if it is empty or nonexisting', async () => {
-    deps.processValues.mockReturnValue(undefined)
+    deps.processValues.mockReturnValue({ originalInput: undefined, allSecrets: {} })
     await bootstrap(deps)
     expect(deps.hfValues).toHaveBeenCalledTimes(0)
   })
@@ -385,7 +388,7 @@ describe('Bootstrapping values', () => {
               { id: 'user2', initialPassword: 'generated-password' },
             ],
           })
-          expect(res).toEqual({
+          expect(res.originalInput).toEqual({
             cluster: { name: 'bla', provider: 'dida' },
             users: [{ id: 'user1', initialPassword: 'existing-password' }, { id: 'user2' }],
           })

src/cmd/bootstrap.ts

@@ -2,7 +2,7 @@ import { randomUUID } from 'crypto'
 import { existsSync } from 'fs'
 import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises'
 import { generate as generatePassword } from 'generate-password'
-import { cloneDeep, get, isEmpty, merge, set } from 'lodash'
+import { cloneDeep, get, merge, set } from 'lodash'
 import { pki } from 'node-forge'
 import path from 'path'
 import { bootstrapGit } from 'src/common/bootstrap'
@@ -14,6 +14,7 @@ import { env, isCli } from 'src/common/envalid'
 import { hfValues } from 'src/common/hf'
 import { createK8sSecret, getDeploymentState, getK8sSecret, secretId } from 'src/common/k8s'
 import { getKmsSettings } from 'src/common/repo'
+import { bootstrapSealedSecrets } from 'src/common/sealed-secrets'
 import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils'
 import { generateSecrets, writeValues } from 'src/common/values'
 import { BasicArguments, setParsedArgs } from 'src/common/yargs'
@@ -84,11 +85,6 @@ export const bootstrapSops = async (
   await deps.writeFile(targetPath, output)
   d.log(`Ready generating sops files. The configuration is written to: ${targetPath}`)
 
-  d.info('Copying sops related files')
-  // add sops related files
-  const file = '.gitattributes'
-  await deps.copyFile(`${rootDir}/.values/${file}`, `${envDir}/${file}`)
-
   // prepare some credential files the first time and crypt some
   if (!exists) {
     if (isCli || env.OTOMI_DEV) {
@@ -291,7 +287,7 @@ export const processValues = async (
     addInitialPasswords,
     addPlatformAdmin,
   },
-): Promise<Record<string, any>> => {
+): Promise<{ originalInput: Record<string, any>; allSecrets: Record<string, any> }> => {
   const d = deps.terminal(`cmd:${cmdName}:processValues`)
   const { VALUES_INPUT } = env
   d.log(`Loading app values from ${VALUES_INPUT}`)
@@ -324,7 +320,7 @@ export const processValues = async (
   // and do some context dependent post processing:
   // to support potential failing chart install we store secrets on cluster
   if (!(env.isDev && env.DISABLE_SYNC)) await deps.createK8sSecret(DEPLOYMENT_PASSWORDS_SECRET, 'otomi', allSecrets)
-  return originalInput
+  return { originalInput, allSecrets }
 }
 
 // create file structure based on file entry
@@ -419,6 +415,7 @@ export const bootstrap = async (
     hfValues,
     writeValues,
     bootstrapSops,
+    bootstrapSealedSecrets,
     migrate,
     encrypt,
     decrypt,
@@ -434,10 +431,10 @@ export const bootstrap = async (
   }
   await deps.copyBasicFiles()
   await deps.migrate()
-  const originalValues = await deps.processValues()
+  const { originalInput, allSecrets } = await deps.processValues()
   await deps.handleFileEntry()
-  await deps.bootstrapSops()
-  await ensureTeamGitOpsDirectories(ENV_DIR, originalValues)
+  await deps.bootstrapSealedSecrets(allSecrets, ENV_DIR, originalInput)
+  await ensureTeamGitOpsDirectories(ENV_DIR, originalInput)
   d.log(`Done bootstrapping values`)
 }
 
@@ -456,7 +453,6 @@ export const module = {
   handler: async (argv: BasicArguments): Promise<void> => {
     setParsedArgs(argv)
     await prepareEnvironment({ skipAllPreChecks: true })
-    await decrypt()
     await bootstrap()
     await bootstrapGit()
   },

src/cmd/commit.ts

@@ -58,7 +58,7 @@ const commitAndPush = async (values: Record<string, any>, branch: string, initia
   const d = terminal(`cmd:${cmdName}:commitAndPush`)
   d.info('Committing values')
   const message = initialInstall ? 'otomi commit' : 'updated values [ci skip]'
-  const { password } = getRepo(values)
+  const { password } = await getRepo(values)
   cd(env.ENV_DIR)
   try {
     try {
@@ -79,8 +79,9 @@ const commitAndPush = async (values: Record<string, any>, branch: string, initia
     }
     await $`git commit -m ${message} --no-verify`.quiet()
   } catch (e) {
-    d.log('commitAndPush error ', e?.message?.replace(password, '****'))
-    return
+    const errorMsg = `commitAndPush error: ${e?.message?.replace(password, '****')}`
+    d.error(errorMsg)
+    throw new Error(errorMsg)
   }
   if (values._derived?.untrustedCA) process.env.GIT_SSL_NO_VERIFY = '1'
   await retry(
@@ -133,10 +134,16 @@ export const commit = async (initialInstall: boolean, overrideArgs?: HelmArgumen
   await validateValues(overrideArgs)
   d.info('Preparing values')
   const values = (await hfValues()) as Record<string, any>
-  const { branch, remote, username, email } = getRepo(values)
+  const { branch, remote, username, email } = await getRepo(values)
   if (initialInstall) {
     // we call this here again, as we might not have completed (happens upon first install):
     await bootstrapGit(values)
+    // Always update the remote URL after bootstrap - the initial bootstrapGit() (called during
+    // the bootstrap phase before install) may have set the URL with unresolved placeholder
+    // passwords because K8s secrets didn't exist yet. Now that secrets are decrypted,
+    // we need to update the URL with the real credentials.
+    cd(env.ENV_DIR)
+    await $`git remote set-url origin ${remote}`.nothrow().quiet()
   } else {
     cd(env.ENV_DIR)
     await setIdentity(username, email)

src/cmd/install.test.ts

@@ -17,6 +17,7 @@ jest.mock('src/common/k8s', () => ({
   applyServerSide: jest.fn(),
   restartOtomiApiDeployment: jest.fn(),
   waitForCRD: jest.fn(),
+  getK8sSecret: jest.fn().mockResolvedValue({ password: 'test', username: 'test' }),
   k8s: {
     app: jest.fn(),
   },
@@ -34,9 +35,41 @@ jest.mock('src/common/hf', () => ({
   HF_DEFAULT_SYNC_ARGS: ['apply', '--sync-args', '--include-needs'],
 }))
 
-jest.mock('zx', () => ({
-  $: jest.fn(),
-  cd: jest.fn(),
+jest.mock('zx', () => {
+  const mockResult = { exitCode: 0, stdout: '', stderr: '' }
+  const createMockProcessPromise = () => {
+    const promise = Promise.resolve(mockResult)
+    const chainable: any = promise
+    chainable.nothrow = jest.fn().mockReturnValue(chainable)
+    chainable.quiet = jest.fn().mockReturnValue(chainable)
+    return chainable
+  }
+  return {
+    $: jest.fn().mockImplementation(() => createMockProcessPromise()),
+    cd: jest.fn(),
+  }
+})
+
+jest.mock('src/common/sealed-secrets', () => ({
+  applySealedSecretManifestsFromDir: jest.fn().mockResolvedValue(undefined),
+  restartSealedSecretsController: jest.fn().mockResolvedValue(undefined),
+  APP_SECRET_OVERRIDES: {
+    'apps.gitea': [
+      {
+        secretName: 'gitea-admin-secret',
+        namespace: 'gitea',
+        data: {
+          username: { valuePath: 'apps.gitea.adminUsername', default: 'otomi-admin' },
+          password: { valuePath: 'apps.gitea.adminPassword' },
+        },
+      },
+      {
+        secretName: 'gitea-db-secret',
+        namespace: 'gitea',
+        data: { username: { static: 'gitea' }, password: { valuePath: 'apps.gitea.postgresqlPassword' } },
+      },
+    ],
+  },
 }))
 
 jest.mock('./commit', () => ({
@@ -106,7 +139,6 @@ describe('Install command', () => {
       stderr: '',
     })
     mockDeps.deployEssential.mockResolvedValue(true)
-    mockDeps.$.mockResolvedValue(undefined)
   })
 
   describe('module configuration', () => {