gitea - migrate storage sections (#1749)

* gitea - migrate storage sections

* remove chown at initial install

* apply code review suggestions

* throw early

* optim9ize

* remove empty lines

* keep old migration logic in

* restore immutable flag

library/ix-dev/community/gitea/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: common
   repository: file://../../../common
-  version: 1.2.2
-digest: sha256:fb077cb81f6acecd5c9e6adc22a18e156f780cd78f27198cdb47810f95364b56
-generated: "2023-11-09T15:40:14.946865509+02:00"
+  version: 1.2.3
+digest: sha256:e6ff49b06bf5d4d159e505ae6d153f36cd46170bb519caf90462cd5caebfd0fb
+generated: "2023-11-15T18:06:15.504956564+02:00"

library/ix-dev/community/gitea/Chart.yaml

@@ -3,9 +3,9 @@ description: Gitea - Git with a cup of tea
 annotations:
   title: Gitea
 type: application
-version: 1.0.23
+version: 1.1.0
 apiVersion: v2
-appVersion: 1.20.5
+appVersion: 1.21.0
 kubeVersion: '>=1.16.0-0'
 maintainers:
   - name: truenas
@@ -14,7 +14,7 @@ maintainers:
 dependencies:
   - name: common
     repository: file://../../../common
-    version: 1.2.2
+    version: 1.2.3
 home: https://gitea.io/en-us
 icon: https://media.sys.truenas.net/apps/gitea/icons/icon.svg
 sources:

library/ix-dev/community/gitea/README.md

@@ -3,13 +3,12 @@
 [Gitea](https://gitea.io/en-us) - Git with a cup of tea
 
 > When application is installed, a container will be launched with **root** privileges.
-> This is required in order to apply the correct permissions to the `gitea` directories.
-> Afterward, the `gitea` container will run as a **non**-root user (Default: `568`).
-> Same applies to the `postgres` container. This will run afterwards as a **non**-root user (`999`).
+> This is required in order to apply the correct permissions to the `postgres` directories.
+> Afterwards `postgres` will run afterwards as a **non**-root user (`999`).
 > On each upgrade, a container will be launched with **root** privileges in order to apply the correct
 > permissions to the `postgres` **backups** directory. Container that performs the backup will run as a **non**-root user (`999`) afterwards.
 > Keep in mind the permissions on the backup directory will be changed to `999:999` on **every** update.
-> But will only be changed once for the `gitea` and `postgres` data directories.
+> But will only be changed once for the `postgres` data directories.
 
-On initial startup a setup wizard will be launched with settings for database, ports and root url prefilled.
-Keep them as they are, fill the Administration section and click on `Install Gitea`.
+On initial startup a setup wizard will be launched with settings for `database`, `ports`, `path`, and `domain` prefilled.
+Keep them as they are, fill anything you want in the optional settings section and click on `Install Gitea`.

library/ix-dev/community/gitea/app-readme.md

@@ -3,13 +3,12 @@
 [Gitea](https://gitea.io/en-us) - Git with a cup of tea
 
 > When application is installed, a container will be launched with **root** privileges.
-> This is required in order to apply the correct permissions to the `gitea` directories.
-> Afterward, the `gitea` container will run as a **non**-root user (Default: `568`).
-> Same applies to the `postgres` container. This will run afterwards as a **non**-root user (`999`).
+> This is required in order to apply the correct permissions to the `postgres` directories.
+> Afterwards `postgres` will run afterwards as a **non**-root user (`999`).
 > On each upgrade, a container will be launched with **root** privileges in order to apply the correct
 > permissions to the `postgres` **backups** directory. Container that performs the backup will run as a **non**-root user (`999`) afterwards.
 > Keep in mind the permissions on the backup directory will be changed to `999:999` on **every** update.
-> But will only be changed once for the `gitea` and `postgres` data directories.
+> But will only be changed once for the `postgres` data directories.
 
 On initial startup a setup wizard will be launched with settings for `database`, `ports`, `path`, and `domain` prefilled.
 Keep them as they are, fill anything you want in the optional settings section and click on `Install Gitea`.

library/ix-dev/community/gitea/ci/basic-values.yaml

@@ -1,16 +1,12 @@
 giteaStorage:
   data:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/data
+    type: pvc
   config:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/config
+    type: pvc
   pgData:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/pgData
+    type: pvc
   pgBackup:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/pgBackup
+    type: emptyDir
 
 giteaNetwork:
   rootURL: http://localhost:30000

library/ix-dev/community/gitea/ci/https-values.yaml

@@ -1,16 +1,12 @@
 giteaStorage:
   data:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/data
+    type: pvc
   config:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/config
+    type: pvc
   pgData:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/pgData
+    type: pvc
   pgBackup:
-    type: hostPath
-    hostPath: /mnt/{{ .Release.Name }}/pgBackup
+    type: emptyDir
 
 giteaNetwork:
   certificateID: 1

library/ix-dev/community/gitea/migrations/migrate

@@ -4,18 +4,44 @@ import os
 import sys
 
 
+def storage_migrate(storage):
+    delete_keys = []
+    if storage['type'] == 'hostPath':
+        # Check if the key exists, if not we have already migrated
+        if not storage.get('hostPath'):
+            return storage
+
+        storage['hostPathConfig'] = {'hostPath': storage['hostPath']}
+        delete_keys.append('hostPath')
+
+    elif storage['type'] == 'ixVolume':
+        # Check if the key exists, if not we have already migrated
+        if not storage.get('datasetName'):
+            return storage
+
+        storage['ixVolumeConfig'] = {'datasetName': storage['datasetName']}
+        delete_keys.append('datasetName')
+
+        # Clean up for some older versions.
+        if storage.get('hostPath'):
+            delete_keys.append('hostPath')
+
+    for key in delete_keys:
+        storage.pop(key, None)
+
+    return storage
+
+
 def migrate(values):
-    storageKey = 'giteaStorage'
+    storage_key = 'giteaStorage'
     storages = ['data', 'config', 'pgData', 'pgBackup']
 
-
     for storage in storages:
-        check_val = values.get(storageKey, {}).get(storage, {})
-        if not isinstance(check_val, dict) or not check_val or check_val.get('type', 'hostPath') == 'hostPath':
-            continue
-
-        values[storageKey][storage] = {key: value for key, value in check_val.items() if key != 'hostPath'}
+        check_val = values.get(storage_key, {}).get(storage, {})
+        if not isinstance(check_val, dict) or not check_val:
+            raise Exception(f'Storage section {storage} is malformed')
 
+        values[storage_key][storage] = storage_migrate(check_val)
 
     return values
 

library/ix-dev/community/gitea/questions.yaml

@@ -156,24 +156,65 @@ questions:
                       description: Host Path (Path that already exists on the system)
                     - value: ixVolume
                       description: ixVolume (Dataset created automatically by the system)
-              - variable: datasetName
-                label: Dataset Name
+              - variable: ixVolumeConfig
+                label: ixVolume Configuration
+                description: The configuration for the ixVolume dataset.
                 schema:
-                  type: string
+                  type: dict
                   show_if: [["type", "=", "ixVolume"]]
-                  required: true
-                  hidden: true
-                  immutable: true
-                  default: data
                   $ref:
                     - "normalize/ixVolume"
-              - variable: hostPath
-                label: Host Path
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        default: false
+                    - variable: datasetName
+                      label: Dataset Name
+                      description: The name of the dataset to use for storage.
+                      schema:
+                        type: string
+                        required: true
+                        immutable: true
+                        hidden: true
+                        default: "data"
+                    - variable: aclEntries
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+              - variable: hostPathConfig
+                label: hostPathConfig
                 schema:
-                  type: hostpath
+                  type: dict
                   show_if: [["type", "=", "hostPath"]]
-                  immutable: true
-                  required: true
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        default: false
+                    - variable: acl
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+                        $ref:
+                          - "normalize/acl"
+                    - variable: hostPath
+                      label: Host Path
+                      description: The host path to use for storage.
+                      schema:
+                        type: hostpath
+                        show_if: [["aclEnable", "=", false]]
+                        immutable: true
+                        required: true
+
         - variable: config
           label: Gitea Configuration Storage
           description: The path to store Gitea configuration storage.
@@ -195,24 +236,65 @@ questions:
                       description: Host Path (Path that already exists on the system)
                     - value: ixVolume
                       description: ixVolume (Dataset created automatically by the system)
-              - variable: datasetName
-                label: Dataset Name
+              - variable: ixVolumeConfig
+                label: ixVolume Configuration
+                description: The configuration for the ixVolume dataset.
                 schema:
-                  type: string
+                  type: dict
                   show_if: [["type", "=", "ixVolume"]]
-                  required: true
-                  hidden: true
-                  immutable: true
-                  default: config
                   $ref:
                     - "normalize/ixVolume"
-              - variable: hostPath
-                label: Host Path
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        default: false
+                    - variable: datasetName
+                      label: Dataset Name
+                      description: The name of the dataset to use for storage.
+                      schema:
+                        type: string
+                        required: true
+                        immutable: true
+                        hidden: true
+                        default: "config"
+                    - variable: aclEntries
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+              - variable: hostPathConfig
+                label: hostPathConfig
                 schema:
-                  type: hostpath
+                  type: dict
                   show_if: [["type", "=", "hostPath"]]
-                  immutable: true
-                  required: true
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        default: false
+                    - variable: acl
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+                        $ref:
+                          - "normalize/acl"
+                    - variable: hostPath
+                      label: Host Path
+                      description: The host path to use for storage.
+                      schema:
+                        type: hostpath
+                        show_if: [["aclEnable", "=", false]]
+                        immutable: true
+                        required: true
+
         - variable: pgData
           label: Gitea Postgres Data Storage
           description: The path to store Gitea Postgres Data.
@@ -234,24 +316,72 @@ questions:
                       description: Host Path (Path that already exists on the system)
                     - value: ixVolume
                       description: ixVolume (Dataset created automatically by the system)
-              - variable: datasetName
-                label: Dataset Name
+              - variable: ixVolumeConfig
+                label: ixVolume Configuration
+                description: The configuration for the ixVolume dataset.
                 schema:
-                  type: string
-                  show_if: [["type", "=", "ixVolume"]]
-                  required: true
+                  type: dict
+                  # Nothing to show for the user
                   hidden: true
-                  immutable: true
-                  default: pgData
+                  show_if: [["type", "=", "ixVolume"]]
                   $ref:
                     - "normalize/ixVolume"
-              - variable: hostPath
-                label: Host Path
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        # Postgres does a CHMOD at startup
+                        # Which fails with ACL
+                        hidden: true
+                        default: false
+                    - variable: datasetName
+                      label: Dataset Name
+                      description: The name of the dataset to use for storage.
+                      schema:
+                        type: string
+                        required: true
+                        immutable: true
+                        hidden: true
+                        default: "pgData"
+                    - variable: aclEntries
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+              - variable: hostPathConfig
+                label: hostPathConfig
                 schema:
-                  type: hostpath
+                  type: dict
                   show_if: [["type", "=", "hostPath"]]
-                  immutable: true
-                  required: true
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        # Postgres does a CHMOD at startup
+                        # Which fails with ACL
+                        hidden: true
+                        default: false
+                    - variable: acl
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+                        $ref:
+                          - "normalize/acl"
+                    - variable: hostPath
+                      label: Host Path
+                      description: The host path to use for storage.
+                      schema:
+                        type: hostpath
+                        show_if: [["aclEnable", "=", false]]
+                        immutable: true
+                        required: true
         - variable: pgBackup
           label: Gitea Postgres Backup Storage
           description: The path to store Gitea Postgres Backup.
@@ -273,24 +403,217 @@ questions:
                       description: Host Path (Path that already exists on the system)
                     - value: ixVolume
                       description: ixVolume (Dataset created automatically by the system)
-              - variable: datasetName
-                label: Dataset Name
+              - variable: ixVolumeConfig
+                label: ixVolume Configuration
+                description: The configuration for the ixVolume dataset.
                 schema:
-                  type: string
-                  show_if: [["type", "=", "ixVolume"]]
-                  required: true
+                  type: dict
+                  # Nothing to show for the user
                   hidden: true
-                  immutable: true
-                  default: pgBackup
+                  show_if: [["type", "=", "ixVolume"]]
                   $ref:
                     - "normalize/ixVolume"
-              - variable: hostPath
-                label: Host Path
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        # Postgres does a CHMOD at startup
+                        # Which fails with ACL
+                        hidden: true
+                        default: false
+                    - variable: datasetName
+                      label: Dataset Name
+                      description: The name of the dataset to use for storage.
+                      schema:
+                        type: string
+                        required: true
+                        immutable: true
+                        hidden: true
+                        default: "pgBackup"
+                    - variable: aclEntries
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+              - variable: hostPathConfig
+                label: hostPathConfig
                 schema:
-                  type: hostpath
+                  type: dict
                   show_if: [["type", "=", "hostPath"]]
-                  immutable: true
-                  required: true
+                  attrs:
+                    - variable: aclEnable
+                      label: Enable ACL
+                      description: Enable ACL for the dataset.
+                      schema:
+                        type: boolean
+                        # Postgres does a CHMOD at startup
+                        # Which fails with ACL
+                        hidden: true
+                        default: false
+                    - variable: acl
+                      label: ACL Configuration
+                      schema:
+                        type: dict
+                        show_if: [["aclEnable", "=", true]]
+                        attrs: []
+                        $ref:
+                          - "normalize/acl"
+                    - variable: hostPath
+                      label: Host Path
+                      description: The host path to use for storage.
+                      schema:
+                        type: hostpath
+                        show_if: [["aclEnable", "=", false]]
+                        immutable: true
+                        required: true
+
+        - variable: additionalStorages
+          label: Additional Storage
+          description: Additional storage for Gitea.
+          schema:
+            type: list
+            default: []
+            items:
+              - variable: storageEntry
+                label: Storage Entry
+                schema:
+                  type: dict
+                  attrs:
+                    - variable: type
+                      label: Type
+                      description: |
+                        ixVolume: Is dataset created automatically by the system.</br>
+                        Host Path: Is a path that already exists on the system.</br>
+                        SMB Share: Is a SMB share that is mounted to a persistent volume claim.
+                      schema:
+                        type: string
+                        required: true
+                        default: "ixVolume"
+                        immutable: true
+                        enum:
+                          - value: "hostPath"
+                            description: Host Path (Path that already exists on the system)
+                          - value: "ixVolume"
+                            description: ixVolume (Dataset created automatically by the system)
+                          - value: "smb-pv-pvc"
+                            description: SMB Share (Mounts a persistent volume claim to a SMB share)
+                    - variable: readOnly
+                      label: Read Only
+                      description: Mount the volume as read only.
+                      schema:
+                        type: boolean
+                        default: false
+                    - variable: mountPath
+                      label: Mount Path
+                      description: The path inside the container to mount the storage.
+                      schema:
+                        type: path
+                        required: true
+                    - variable: hostPathConfig
+                      label: hostPathConfig
+                      schema:
+                        type: dict
+                        show_if: [["type", "=", "hostPath"]]
+                        attrs:
+                          - variable: aclEnable
+                            label: Enable ACL
+                            description: Enable ACL for the dataset.
+                            schema:
+                              type: boolean
+                              default: false
+                          - variable: acl
+                            label: ACL Configuration
+                            schema:
+                              type: dict
+                              show_if: [["aclEnable", "=", true]]
+                              attrs: []
+                              $ref:
+                                - "normalize/acl"
+                          - variable: hostPath
+                            label: Host Path
+                            description: The host path to use for storage.
+                            schema:
+                              type: hostpath
+                              show_if: [["aclEnable", "=", false]]
+                              immutable: true
+                              required: true
+                    - variable: ixVolumeConfig
+                      label: ixVolume Configuration
+                      description: The configuration for the ixVolume dataset.
+                      schema:
+                        type: dict
+                        show_if: [["type", "=", "ixVolume"]]
+                        $ref:
+                          - "normalize/ixVolume"
+                        attrs:
+                          - variable: aclEnable
+                            label: Enable ACL
+                            description: Enable ACL for the dataset.
+                            schema:
+                              type: boolean
+                              default: false
+                          - variable: datasetName
+                            label: Dataset Name
+                            description: The name of the dataset to use for storage.
+                            schema:
+                              type: string
+                              required: true
+                              immutable: true
+                              default: "storage_entry"
+                          - variable: aclEntries
+                            label: ACL Configuration
+                            schema:
+                              type: dict
+                              show_if: [["aclEnable", "=", true]]
+                              attrs: []
+                    - variable: smbConfig
+                      label: SMB Share Configuration
+                      description: The configuration for the SMB Share.
+                      schema:
+                        type: dict
+                        show_if: [["type", "=", "smb-pv-pvc"]]
+                        attrs:
+                          - variable: server
+                            label: Server
+                            description: The server for the SMB share.
+                            schema:
+                              type: string
+                              required: true
+                          - variable: share
+                            label: Share
+                            description: The share name for the SMB share.
+                            schema:
+                              type: string
+                              required: true
+                          - variable: domain
+                            label: Domain (Optional)
+                            description: The domain for the SMB share.
+                            schema:
+                              type: string
+                          - variable: username
+                            label: Username
+                            description: The username for the SMB share.
+                            schema:
+                              type: string
+                              required: true
+                          - variable: password
+                            label: Password
+                            description: The password for the SMB share.
+                            schema:
+                              type: string
+                              required: true
+                              private: true
+                          - variable: size
+                            label: Size (in Gi)
+                            description: The size of the volume quota.
+                            schema:
+                              type: int
+                              required: true
+                              min: 1
+                              default: 1
 
   - variable: resources
     label: ""

library/ix-dev/community/gitea/templates/_gitea.tpl

@@ -47,10 +47,6 @@ workload:
               path: /api/healthz
               port: {{ .Values.giteaNetwork.webPort }}
       initContainers:
-      {{- include "ix.v1.common.app.permissions" (dict "containerName" "01-permissions"
-                                                        "UID" .Values.giteaRunAs.user
-                                                        "GID" .Values.giteaRunAs.group
-                                                        "type" "install") | nindent 8 }}
       {{- include "ix.v1.common.app.postgresWait" (dict "name" "postgres-wait"
                                                         "secretName" "postgres-creds") | nindent 8 }}
 {{/* Service */}}
@@ -77,9 +73,8 @@ service:
 persistence:
   data:
     enabled: true
-    type: {{ .Values.giteaStorage.data.type }}
-    datasetName: {{ .Values.giteaStorage.data.datasetName | default "" }}
-    hostPath: {{ .Values.giteaStorage.data.hostPath | default "" }}
+    {{- include "gitea.storage.ci.migration" (dict "storage" .Values.giteaStorage.data) }}
+    {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.giteaStorage.data) | nindent 4 }}
     targetSelector:
       gitea:
         gitea:
@@ -88,9 +83,8 @@ persistence:
           mountPath: /mnt/directories/data
   config:
     enabled: true
-    type: {{ .Values.giteaStorage.config.type }}
-    datasetName: {{ .Values.giteaStorage.config.datasetName | default "" }}
-    hostPath: {{ .Values.giteaStorage.config.hostPath | default "" }}
+    {{- include "gitea.storage.ci.migration" (dict "storage" .Values.giteaStorage.config) }}
+    {{- include "ix.v1.common.app.storageOptions" (dict "storage" .Values.giteaStorage.config) | nindent 4 }}
     targetSelector:
       gitea:
         gitea:
@@ -104,6 +98,17 @@ persistence:
       gitea:
         gitea:
           mountPath: /tmp/gitea
+
+  {{- range $idx, $storage := .Values.giteaStorage.additionalStorages }}
+  {{ printf "gitea-%v:" (int $idx) }}
+    enabled: true
+    {{- include "ix.v1.common.app.storageOptions" (dict "storage" $storage) | nindent 4 }}
+    targetSelector:
+      gitea:
+        gitea:
+          mountPath: {{ $storage.mountPath }}
+  {{- end }}
+
   {{ if .Values.giteaNetwork.certificateID }}
   cert:
     enabled: true
@@ -122,3 +127,14 @@ persistence:
           readOnly: true
   {{ end }}
 {{- end -}}
+
+
+{{/* TODO: Remove on the next version bump, eg 1.1.0+ */}}
+{{- define "gitea.storage.ci.migration" -}}
+  {{- $storage := .storage -}}
+
+  {{- if $storage.hostPath -}}
+    {{- $_ := set $storage "hostPathConfig" dict -}}
+    {{- $_ := set $storage.hostPathConfig "hostPath" $storage.hostPath -}}
+  {{- end -}}
+{{- end -}}

library/ix-dev/community/gitea/templates/_postgres.tpl

@@ -7,6 +7,8 @@ workload:
 service:
   {{- include "ix.v1.common.app.postgresService" $ | nindent 2 }}
 
+{{- include "gitea.storage.ci.migration" (dict "storage" .Values.giteaStorage.pgData) }}
+{{- include "gitea.storage.ci.migration" (dict "storage" .Values.giteaStorage.pgBackup) }}
 {{/* Persistence */}}
 persistence:
   {{- include "ix.v1.common.app.postgresPersistence"

library/ix-dev/community/gitea/values.yaml

@@ -1,7 +1,7 @@
 image:
   repository: gitea/gitea
   pullPolicy: IfNotPresent
-  tag: 1.20.5-rootless
+  tag: 1.21.0-rootless
 
 resources:
   limits:
@@ -28,16 +28,21 @@ giteaRunAs:
 giteaStorage:
   data:
     type: ixVolume
-    datasetName: data
+    ixVolumeConfig:
+      datasetName: data
   config:
     type: ixVolume
-    datasetName: config
+    ixVolumeConfig:
+      datasetName: config
   pgData:
     type: ixVolume
-    datasetName: pgData
+    ixVolumeConfig:
+      datasetName: pgData
   pgBackup:
     type: ixVolume
-    datasetName: pgBackup
+    ixVolumeConfig:
+      datasetName: pgBackup
+  additionalStorages: []
 
 notes:
   custom: |