feat(azblob): add azure blob storage provider

.github/workflows/ci-cd.yml

@@ -45,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        storage-driver: [filesystem, s3, gcs]
+        storage-driver: [filesystem, s3, gcs, azblob]
         db-driver: [postgres, mysql, sqlite]
     steps:
       - name: pnpm install

install/kubernetes/github-actions-cache-server/templates/_helpers.tpl

@@ -193,6 +193,21 @@ Generate environment variables from config values.
   value: {{ .endpoint | quote }}
 {{- end }}
 {{- end }}
+{{- else if eq .Values.config.storage.driver "azblob" }}
+{{- with .Values.config.storage.azblob }}
+{{- if .account }}
+- name: STORAGE_AZBLOB_ACCOUNT
+  value: {{ .account | quote }}
+{{- end }}
+{{- if .container }}
+- name: STORAGE_AZBLOB_CONTAINER
+  value: {{ .container | quote }}
+{{- end }}
+{{- if .endpoint }}
+- name: STORAGE_AZBLOB_ENDPOINT
+  value: {{ .endpoint | quote }}
+{{- end }}
+{{- end }}
 {{- end }}
 {{/* Database driver */}}
 - name: DB_DRIVER

install/kubernetes/github-actions-cache-server/values.yaml

@@ -67,7 +67,7 @@ config:
   # -- Storage driver configuration
   # See https://gha-cache-server.falcondev.io/storage-drivers
   storage:
-    # -- Storage driver to use: "filesystem", "s3", or "gcs"
+    # -- Storage driver to use: "filesystem", "s3", "gcs", or "azblob"
     driver: filesystem
 
     # -- Filesystem storage driver settings
@@ -102,6 +102,15 @@ config:
       # -- Custom GCS API endpoint
       # endpoint: ''
 
+    # -- Azure Blob Storage driver settings
+    azblob:
+      # -- Azure Storage account name (required when driver is "azblob")
+      # account: ''
+      # -- Azure Blob container name (required when driver is "azblob")
+      # container: ''
+      # -- Custom Azure Blob Storage endpoint URL (optional, defaults URL created using account name)
+      # endpoint: ''
+
   # -- Database driver configuration
   # See https://gha-cache-server.falcondev.io/database-drivers
   db:
@@ -147,7 +156,7 @@ config:
 
 # -- Name of an existing Kubernetes Secret containing sensitive environment variables.
 # Use this to avoid storing secrets in values.yaml. The secret keys should match
-# the env var names (e.g., AWS_SECRET_ACCESS_KEY, DB_POSTGRES_PASSWORD, DB_POSTGRES_URL, etc.)
+# the env var names (e.g., AWS_SECRET_ACCESS_KEY, DB_POSTGRES_PASSWORD, DB_POSTGRES_URL, STORAGE_AZBLOB_CONNECTION_STRING  etc.)
 existingSecret: ''
 
 serviceAccount:

lib/schemas.ts

@@ -20,6 +20,13 @@ export const envStorageDriverSchema = type.or(
     'STORAGE_GCS_SERVICE_ACCOUNT_KEY?': 'string',
     'STORAGE_GCS_ENDPOINT?': 'string.url',
   },
+  {
+    'STORAGE_DRIVER': type.unit('azblob'),
+    'STORAGE_AZBLOB_ACCOUNT': 'string',
+    'STORAGE_AZBLOB_CONTAINER': 'string',
+    'STORAGE_AZBLOB_CONNECTION_STRING?': 'string',
+    'STORAGE_AZBLOB_ENDPOINT?': 'string.url',
+  },
 )
 export const envDbDriverSchema = type.or(
   type.or(

lib/storage.ts

@@ -22,6 +22,12 @@ import {
 } from '@aws-sdk/client-s3'
 import { Upload as S3Upload } from '@aws-sdk/lib-storage'
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { DefaultAzureCredential } from '@azure/identity'
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  generateBlobSASQueryParameters,
+} from '@azure/storage-blob'
 import { Storage as GcsClient } from '@google-cloud/storage'
 import { NodeHttpHandler } from '@smithy/node-http-handler'
 import { sql } from 'kysely'
@@ -76,6 +82,7 @@ export class Storage {
       .with({ STORAGE_DRIVER: 's3' }, S3Adapter.fromEnv)
       .with({ STORAGE_DRIVER: 'filesystem' }, FileSystemAdapter.fromEnv)
       .with({ STORAGE_DRIVER: 'gcs' }, GcsAdapter.fromEnv)
+      .with({ STORAGE_DRIVER: 'azblob' }, AzBlobAdapter.fromEnv)
       .exhaustive()
   }
 
@@ -1257,3 +1264,163 @@ class GcsAdapter implements StorageAdapter {
       .then((res) => res[0])
   }
 }
+
+class AzBlobAdapter implements StorageAdapter {
+  static async fromEnv(env: Extract<Env, { STORAGE_DRIVER: 'azblob' }>) {
+    const account = env.STORAGE_AZBLOB_ACCOUNT
+    const container = env.STORAGE_AZBLOB_CONTAINER
+
+    const client = env.STORAGE_AZBLOB_CONNECTION_STRING
+      ? BlobServiceClient.fromConnectionString(env.STORAGE_AZBLOB_CONNECTION_STRING)
+      : new BlobServiceClient(
+          env.STORAGE_AZBLOB_ENDPOINT ?? `https://${account}.blob.core.windows.net`,
+          new DefaultAzureCredential(),
+        )
+
+    const containerClient = client.getContainerClient(container)
+    await containerClient.createIfNotExists()
+
+    return new AzBlobAdapter({
+      client,
+      account,
+      container,
+    })
+  }
+
+  private client
+  private account
+  private container
+  private keyPrefix = 'gh-actions-cache'
+
+  constructor({
+    client,
+    account,
+    container,
+  }: {
+    client: BlobServiceClient
+    account: string
+    container: string
+  }) {
+    this.client = client
+    this.account = account
+    this.container = container
+  }
+
+  private get containerClient() {
+    return this.client.getContainerClient(this.container)
+  }
+
+  private blobKey(objectName: string) {
+    return `${this.keyPrefix}/${objectName}`
+  }
+
+  async createDownloadStream(objectName: string): Promise<Readable> {
+    const blockBlobClient = this.containerClient.getBlockBlobClient(this.blobKey(objectName))
+    const response = await blockBlobClient.download()
+    if (!response.readableStreamBody) throw new Error(`No stream for blob "${objectName}"`)
+    return Readable.from(response.readableStreamBody)
+  }
+
+  async uploadStream(objectName: string, stream: AsyncIterable<Uint8Array>): Promise<void> {
+    const blockBlobClient = this.containerClient.getBlockBlobClient(this.blobKey(objectName))
+    // TODO: consider blockSize / concurrency tuning similar to S3Upload options
+    await blockBlobClient.uploadStream(Readable.from(stream))
+  }
+
+  async objectExists(objectName: string): Promise<boolean> {
+    return this.containerClient.getBlobClient(this.blobKey(objectName)).exists()
+  }
+
+  async deleteByPrefix(prefix: string): Promise<StorageDeletion> {
+    // Azure caps a batch at 256 subrequests - align LIST paging with that so each page = one batch.
+    const BATCH_SIZE = 256
+    const deleted = { objects: 0, bytes: 0 }
+    const batchClient = this.containerClient.getBlobBatchClient()
+
+    const pages = this.containerClient.listBlobsFlat({ prefix }).byPage({ maxPageSize: BATCH_SIZE })
+
+    for await (const page of pages) {
+      const blobs = page.segment.blobItems
+      if (blobs.length === 0) continue
+
+      const clients = blobs.map((blob) => {
+        deleted.objects += 1
+        deleted.bytes += blob.properties.contentLength ?? 0
+        return this.containerClient.getBlobClient(blob.name)
+      })
+
+      await batchClient.deleteBlobs(clients)
+    }
+
+    return deleted
+  }
+
+  async deleteFolder(folderName: string): Promise<StorageDeletion> {
+    return this.deleteByPrefix(`${this.blobKey(folderName)}/`)
+  }
+
+  async clear(): Promise<void> {
+    await this.deleteByPrefix(this.blobKey(''))
+  }
+
+  async countFilesInFolder(folderName: string): Promise<number> {
+    let count = 0
+    const blobs = this.containerClient.listBlobsFlat({
+      prefix: `${this.blobKey(folderName)}/`,
+    })
+    for await (const _ of blobs) {
+      count++
+    }
+    return count
+  }
+
+  async getFolderSize(folderName: string): Promise<number> {
+    const blobs = this.containerClient.listBlobsFlat({
+      prefix: `${this.blobKey(folderName)}/`,
+    })
+    let size = 0
+    for await (const blob of blobs) {
+      size += blob.properties.contentLength ?? 0
+    }
+    return size
+  }
+
+  async listStorageFolders(): Promise<StorageFolder[]> {
+    const folders = new Map<string, StorageFolder>()
+    const prefix = this.blobKey('')
+
+    const blobs = this.containerClient.listBlobsFlat({ prefix })
+    for await (const blob of blobs) {
+      const relativeName = blob.name.slice(prefix.length)
+      const folderName = relativeName.split('/', 1)[0]
+      if (!folderName) continue
+
+      const size = blob.properties.contentLength ?? 0
+      const updatedAt = blob.properties.lastModified?.getTime() ?? 0
+      accumulateFolder(folders, folderName, size, updatedAt)
+    }
+
+    return [...folders.values()]
+  }
+
+  async createDownloadUrl(objectName: string, expiresAt: number): Promise<string> {
+    const startsOn = new Date()
+    const expiresOn = new Date(expiresAt)
+
+    const delegationKey = await this.client.getUserDelegationKey(startsOn, expiresOn)
+
+    const sasParams = generateBlobSASQueryParameters(
+      {
+        containerName: this.container,
+        blobName: this.blobKey(objectName),
+        permissions: BlobSASPermissions.parse('r'),
+        startsOn,
+        expiresOn,
+      },
+      delegationKey,
+      this.account,
+    )
+
+    return `https://${this.account}.blob.core.windows.net/${this.container}/${this.blobKey(objectName)}?${sasParams.toString()}`
+  }
+}

package.json

@@ -15,7 +15,7 @@
     "lint:fix": "eslint --fix --cache . && prettier --write --cache .",
     "type-check": "tsc --noEmit",
     "test:run": "DEBUG=true vitest run",
-    "test:matrix": "for db in sqlite postgres mysql; do for st in filesystem s3 gcs; do echo \"### $db + $st\"; VITEST_DB_DRIVER=$db VITEST_STORAGE_DRIVER=$st pnpm run test:run || exit 1; done; done"
+    "test:matrix": "for db in sqlite postgres mysql; do for st in filesystem s3 gcs azblob; do echo \"### $db + $st\"; VITEST_DB_DRIVER=$db VITEST_STORAGE_DRIVER=$st pnpm run test:run || exit 1; done; done"
   },
   "changelogithub": {
     "extends": "gh:falcondev-it/configs/changelogithub"
@@ -25,6 +25,8 @@
     "@aws-sdk/client-s3": "^3.1085.0",
     "@aws-sdk/lib-storage": "^3.1085.0",
     "@aws-sdk/s3-request-presigner": "^3.1085.0",
+    "@azure/identity": "^4.13.1",
+    "@azure/storage-blob": "^12.33.0",
     "@google-cloud/storage": "^7.21.0",
     "@orpc/client": "^1.14.7",
     "@orpc/json-schema": "^1.14.7",