Metaxy + DuckLake¶

Experimental

This functionality is experimental.

DuckLake is a modern LakeHouse which uses a relational database as metadata catalog.

Currently, there is only one production-ready implementation of DuckLake - via DuckDB, and the built-in DuckDBMetadataStore can be configured to use DuckLake as its storage backend. Learn more about the DuckDB integration here.

Configuration¶

There are two main parts that configure DuckLake: a catalog (where the transaction log and other metadata is stored) and a storage (where the data files (1) live).

Parquet files

Each piece of configuration that manages secrets (e.g. PostgreSQL, S3, R2, GCS) requires a secret_name parameter. Metaxy uses this name to either create a new DuckDB secret (when inline credentials are provided) or reference a pre-existing one (when only the name is given).

Tip

To use the credential chain (IAM roles, environment variables, etc.) instead of static S3 credentials, set secret_parameters = { provider = "credential_chain" }. Learn more in DuckDB docs.

Example Configuration

[stores.dev]
type = "metaxy.ext.duckdb.DuckDBMetadataStore"

[stores.dev.config.ducklake.catalog]
type = "postgres"
secret_name = "my_pg_secret"
host = "localhost"
port = 5432
database = "ducklake_meta"
user = "ducklake"
password = "changeme"

[stores.dev.config.ducklake.storage]
type = "s3"
secret_name = "my_s3_secret"
bucket = "my-ducklake-bucket"
key_id = "AKIA..."
secret = "..."
region = "eu-central-1"

See the DuckLake example to learn more.

DuckLake attachment configuration for a DuckDB connection.

Show JSON schema:

{
  "$defs": {
    "DuckDBCatalogConfig": {
      "description": "DuckDB file-based metadata backend for [DuckLake](https://ducklake.select/).",
      "properties": {
        "type": {
          "const": "duckdb",
          "default": "duckdb",
          "title": "Type",
          "type": "string"
        },
        "uri": {
          "title": "Uri",
          "type": "string"
        }
      },
      "required": [
        "uri"
      ],
      "title": "DuckDBCatalogConfig",
      "type": "object"
    },
    "GCSStorageConfig": {
      "description": "Google Cloud Storage backend for [DuckLake](https://ducklake.select/).\n\nUses the DuckDB [`TYPE GCS`](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api#gcs-secrets) secret.",
      "properties": {
        "type": {
          "const": "gcs",
          "default": "gcs",
          "title": "Type",
          "type": "string"
        },
        "key_id": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Key Id"
        },
        "secret": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret"
        },
        "data_path": {
          "title": "Data Path",
          "type": "string"
        },
        "secret_name": {
          "title": "Secret Name",
          "type": "string"
        },
        "secret_parameters": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret Parameters"
        }
      },
      "required": [
        "data_path",
        "secret_name"
      ],
      "title": "GCSStorageConfig",
      "type": "object"
    },
    "LocalStorageConfig": {
      "description": "Local filesystem storage backend for DuckLake.",
      "properties": {
        "type": {
          "const": "local",
          "default": "local",
          "title": "Type",
          "type": "string"
        },
        "path": {
          "title": "Path",
          "type": "string"
        }
      },
      "required": [
        "path"
      ],
      "title": "LocalStorageConfig",
      "type": "object"
    },
    "MotherDuckCatalogConfig": {
      "description": "[MotherDuck](https://motherduck.com/)-managed metadata backend for [DuckLake](https://ducklake.select/).",
      "properties": {
        "type": {
          "const": "motherduck",
          "default": "motherduck",
          "title": "Type",
          "type": "string"
        },
        "database": {
          "title": "Database",
          "type": "string"
        },
        "region": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "AWS region of the MotherDuck-managed S3 storage (e.g. 'eu-central-1').",
          "title": "Region"
        }
      },
      "required": [
        "database"
      ],
      "title": "MotherDuckCatalogConfig",
      "type": "object"
    },
    "PostgresCatalogConfig": {
      "description": "PostgreSQL metadata backend for [DuckLake](https://ducklake.select/).",
      "properties": {
        "type": {
          "const": "postgres",
          "default": "postgres",
          "title": "Type",
          "type": "string"
        },
        "database": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Database"
        },
        "user": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "User"
        },
        "password": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Password"
        },
        "host": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Host"
        },
        "port": {
          "default": 5432,
          "title": "Port",
          "type": "integer"
        },
        "secret_name": {
          "title": "Secret Name",
          "type": "string"
        },
        "secret_parameters": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret Parameters"
        }
      },
      "required": [
        "secret_name"
      ],
      "title": "PostgresCatalogConfig",
      "type": "object"
    },
    "R2StorageConfig": {
      "description": "Cloudflare R2 storage backend for [DuckLake](https://ducklake.select/).\n\nUses the DuckDB [`TYPE R2`](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api#r2-secrets) secret.",
      "properties": {
        "type": {
          "const": "r2",
          "default": "r2",
          "title": "Type",
          "type": "string"
        },
        "key_id": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Key Id"
        },
        "secret": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret"
        },
        "account_id": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Account Id"
        },
        "data_path": {
          "title": "Data Path",
          "type": "string"
        },
        "secret_name": {
          "title": "Secret Name",
          "type": "string"
        },
        "secret_parameters": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret Parameters"
        }
      },
      "required": [
        "data_path",
        "secret_name"
      ],
      "title": "R2StorageConfig",
      "type": "object"
    },
    "S3StorageConfig": {
      "description": "[S3 storage](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api) backend for DuckLake.",
      "properties": {
        "type": {
          "const": "s3",
          "default": "s3",
          "title": "Type",
          "type": "string"
        },
        "key_id": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Key Id"
        },
        "secret": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret"
        },
        "endpoint": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Endpoint"
        },
        "bucket": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Bucket"
        },
        "prefix": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Prefix"
        },
        "region": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Region"
        },
        "url_style": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Url Style"
        },
        "use_ssl": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Use Ssl"
        },
        "scope": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Scope"
        },
        "data_path": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Data Path"
        },
        "secret_name": {
          "title": "Secret Name",
          "type": "string"
        },
        "secret_parameters": {
          "anyOf": [
            {
              "additionalProperties": true,
              "type": "object"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Secret Parameters"
        }
      },
      "required": [
        "secret_name"
      ],
      "title": "S3StorageConfig",
      "type": "object"
    },
    "SQLiteCatalogConfig": {
      "description": "SQLite file-based metadata backend for [DuckLake](https://ducklake.select/).",
      "properties": {
        "type": {
          "const": "sqlite",
          "default": "sqlite",
          "title": "Type",
          "type": "string"
        },
        "uri": {
          "title": "Uri",
          "type": "string"
        }
      },
      "required": [
        "uri"
      ],
      "title": "SQLiteCatalogConfig",
      "type": "object"
    }
  },
  "description": "[DuckLake](https://ducklake.select/) attachment configuration for a DuckDB connection.",
  "properties": {
    "catalog": {
      "description": "Metadata catalog backend (DuckDB, SQLite, PostgreSQL, or MotherDuck).",
      "discriminator": {
        "mapping": {
          "duckdb": "#/$defs/DuckDBCatalogConfig",
          "motherduck": "#/$defs/MotherDuckCatalogConfig",
          "postgres": "#/$defs/PostgresCatalogConfig",
          "sqlite": "#/$defs/SQLiteCatalogConfig"
        },
        "propertyName": "type"
      },
      "oneOf": [
        {
          "$ref": "#/$defs/DuckDBCatalogConfig"
        },
        {
          "$ref": "#/$defs/SQLiteCatalogConfig"
        },
        {
          "$ref": "#/$defs/PostgresCatalogConfig"
        },
        {
          "$ref": "#/$defs/MotherDuckCatalogConfig"
        }
      ],
      "title": "Catalog"
    },
    "storage": {
      "anyOf": [
        {
          "discriminator": {
            "mapping": {
              "gcs": "#/$defs/GCSStorageConfig",
              "local": "#/$defs/LocalStorageConfig",
              "r2": "#/$defs/R2StorageConfig",
              "s3": "#/$defs/S3StorageConfig"
            },
            "propertyName": "type"
          },
          "oneOf": [
            {
              "$ref": "#/$defs/LocalStorageConfig"
            },
            {
              "$ref": "#/$defs/S3StorageConfig"
            },
            {
              "$ref": "#/$defs/R2StorageConfig"
            },
            {
              "$ref": "#/$defs/GCSStorageConfig"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Data storage backend (local filesystem, S3, R2, or GCS). Not required for MotherDuck.",
      "title": "Storage"
    },
    "alias": {
      "default": "ducklake",
      "description": "DuckDB catalog alias for the attached DuckLake database.",
      "title": "Alias",
      "type": "string"
    },
    "attach_options": {
      "additionalProperties": true,
      "description": "Extra [DuckLake](https://ducklake.select/) ATTACH options (e.g., api_version, override_data_path).",
      "title": "Attach Options",
      "type": "object"
    },
    "data_inlining_row_limit": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Store inserts smaller than this row count directly in the metadata catalog instead of creating Parquet files.",
      "title": "Data Inlining Row Limit"
    }
  },
  "required": [
    "catalog"
  ],
  "title": "DuckLakeConfig",
  "type": "object"
}

catalog `pydantic-field` ¶

catalog: (
    DuckDBCatalogConfig
    | SQLiteCatalogConfig
    | PostgresCatalogConfig
    | MotherDuckCatalogConfig
)

storage `pydantic-field` ¶

storage: (
    LocalStorageConfig
    | S3StorageConfig
    | R2StorageConfig
    | GCSStorageConfig
    | None
) = None

Data storage backend (local filesystem, S3, R2, or GCS). Not required for MotherDuck.

alias `pydantic-field` ¶

alias: str = 'ducklake'

DuckDB catalog alias for the attached DuckLake database.

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake]
alias = "ducklake"

[tool.metaxy.stores.dev.config.ducklake]
alias = "ducklake"

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__ALIAS=ducklake

attach_options `pydantic-field` ¶

attach_options: dict[str, Any]

Extra DuckLake ATTACH options (e.g., api_version, override_data_path).

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake]
attach_options = {}

[tool.metaxy.stores.dev.config.ducklake]
attach_options = {}

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__ATTACH_OPTIONS={}

data_inlining_row_limit `pydantic-field` ¶

data_inlining_row_limit: int | None = None

Store inserts smaller than this row count directly in the metadata catalog instead of creating Parquet files.

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake]
data_inlining_row_limit = 0

[tool.metaxy.stores.dev.config.ducklake]
data_inlining_row_limit = 0

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__DATA_INLINING_ROW_LIMIT=...

Catalog Backends¶

DuckDB file-based metadata backend for DuckLake.

Show JSON schema:

{
  "description": "DuckDB file-based metadata backend for [DuckLake](https://ducklake.select/).",
  "properties": {
    "type": {
      "const": "duckdb",
      "default": "duckdb",
      "title": "Type",
      "type": "string"
    },
    "uri": {
      "title": "Uri",
      "type": "string"
    }
  },
  "required": [
    "uri"
  ],
  "title": "DuckDBCatalogConfig",
  "type": "object"
}

uri `pydantic-field` ¶

uri: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
uri = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
uri = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__URI=...

SQLite file-based metadata backend for DuckLake.

Show JSON schema:

{
  "description": "SQLite file-based metadata backend for [DuckLake](https://ducklake.select/).",
  "properties": {
    "type": {
      "const": "sqlite",
      "default": "sqlite",
      "title": "Type",
      "type": "string"
    },
    "uri": {
      "title": "Uri",
      "type": "string"
    }
  },
  "required": [
    "uri"
  ],
  "title": "SQLiteCatalogConfig",
  "type": "object"
}

uri `pydantic-field` ¶

uri: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
uri = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
uri = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__URI=...

PostgreSQL metadata backend for DuckLake.

Show JSON schema:

{
  "description": "PostgreSQL metadata backend for [DuckLake](https://ducklake.select/).",
  "properties": {
    "type": {
      "const": "postgres",
      "default": "postgres",
      "title": "Type",
      "type": "string"
    },
    "database": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Database"
    },
    "user": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "User"
    },
    "password": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Password"
    },
    "host": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Host"
    },
    "port": {
      "default": 5432,
      "title": "Port",
      "type": "integer"
    },
    "secret_name": {
      "title": "Secret Name",
      "type": "string"
    },
    "secret_parameters": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret Parameters"
    }
  },
  "required": [
    "secret_name"
  ],
  "title": "PostgresCatalogConfig",
  "type": "object"
}

database `pydantic-field` ¶

database: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
database = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
database = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__DATABASE=...

user `pydantic-field` ¶

user: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
user = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
user = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__USER=...

password `pydantic-field` ¶

password: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
password = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
password = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__PASSWORD=...

host `pydantic-field` ¶

host: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
host = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
host = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__HOST=...

port `pydantic-field` ¶

port: int = 5432

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
port = 5432

[tool.metaxy.stores.dev.config.ducklake.catalog]
port = 5432

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__PORT=5432

secret_name `pydantic-field` ¶

secret_name: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
secret_name = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
secret_name = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__SECRET_NAME=...

secret_parameters `pydantic-field` ¶

secret_parameters: dict[str, Any] | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
secret_parameters = {}

[tool.metaxy.stores.dev.config.ducklake.catalog]
secret_parameters = {}

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__SECRET_PARAMETERS=...

MotherDuck-managed metadata backend for DuckLake.

Show JSON schema:

{
  "description": "[MotherDuck](https://motherduck.com/)-managed metadata backend for [DuckLake](https://ducklake.select/).",
  "properties": {
    "type": {
      "const": "motherduck",
      "default": "motherduck",
      "title": "Type",
      "type": "string"
    },
    "database": {
      "title": "Database",
      "type": "string"
    },
    "region": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "AWS region of the MotherDuck-managed S3 storage (e.g. 'eu-central-1').",
      "title": "Region"
    }
  },
  "required": [
    "database"
  ],
  "title": "MotherDuckCatalogConfig",
  "type": "object"
}

database `pydantic-field` ¶

database: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
database = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
database = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__DATABASE=...

region `pydantic-field` ¶

region: str | None = None

AWS region of the MotherDuck-managed S3 storage (e.g. 'eu-central-1').

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.catalog]
region = "..."

[tool.metaxy.stores.dev.config.ducklake.catalog]
region = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__CATALOG__REGION=...

Storage Backends¶

Local filesystem storage backend for DuckLake.

Show JSON schema:

{
  "description": "Local filesystem storage backend for DuckLake.",
  "properties": {
    "type": {
      "const": "local",
      "default": "local",
      "title": "Type",
      "type": "string"
    },
    "path": {
      "title": "Path",
      "type": "string"
    }
  },
  "required": [
    "path"
  ],
  "title": "LocalStorageConfig",
  "type": "object"
}

path `pydantic-field` ¶

path: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
path = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
path = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__PATH=...

S3 storage backend for DuckLake.

Show JSON schema:

{
  "description": "[S3 storage](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api) backend for DuckLake.",
  "properties": {
    "type": {
      "const": "s3",
      "default": "s3",
      "title": "Type",
      "type": "string"
    },
    "key_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Key Id"
    },
    "secret": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret"
    },
    "endpoint": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Endpoint"
    },
    "bucket": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Bucket"
    },
    "prefix": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Prefix"
    },
    "region": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Region"
    },
    "url_style": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Url Style"
    },
    "use_ssl": {
      "anyOf": [
        {
          "type": "boolean"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Use Ssl"
    },
    "scope": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Scope"
    },
    "data_path": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Data Path"
    },
    "secret_name": {
      "title": "Secret Name",
      "type": "string"
    },
    "secret_parameters": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret Parameters"
    }
  },
  "required": [
    "secret_name"
  ],
  "title": "S3StorageConfig",
  "type": "object"
}

key_id `pydantic-field` ¶

key_id: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
key_id = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
key_id = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__KEY_ID=...

secret `pydantic-field` ¶

secret: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET=...

endpoint `pydantic-field` ¶

endpoint: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
endpoint = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
endpoint = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__ENDPOINT=...

bucket `pydantic-field` ¶

bucket: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
bucket = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
bucket = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__BUCKET=...

prefix `pydantic-field` ¶

prefix: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
prefix = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
prefix = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__PREFIX=...

region `pydantic-field` ¶

region: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
region = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
region = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__REGION=...

url_style `pydantic-field` ¶

url_style: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
url_style = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
url_style = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__URL_STYLE=...

use_ssl `pydantic-field` ¶

use_ssl: bool | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
use_ssl = false

[tool.metaxy.stores.dev.config.ducklake.storage]
use_ssl = false

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__USE_SSL=...

scope `pydantic-field` ¶

scope: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
scope = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
scope = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SCOPE=...

data_path `pydantic-field` ¶

data_path: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
data_path = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
data_path = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__DATA_PATH=...

secret_name `pydantic-field` ¶

secret_name: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_name = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_name = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_NAME=...

secret_parameters `pydantic-field` ¶

secret_parameters: dict[str, Any] | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_parameters = {}

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_parameters = {}

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_PARAMETERS=...

Cloudflare R2 storage backend for DuckLake.

Uses the DuckDB TYPE R2 secret.

Show JSON schema:

{
  "description": "Cloudflare R2 storage backend for [DuckLake](https://ducklake.select/).\n\nUses the DuckDB [`TYPE R2`](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api#r2-secrets) secret.",
  "properties": {
    "type": {
      "const": "r2",
      "default": "r2",
      "title": "Type",
      "type": "string"
    },
    "key_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Key Id"
    },
    "secret": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret"
    },
    "account_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Account Id"
    },
    "data_path": {
      "title": "Data Path",
      "type": "string"
    },
    "secret_name": {
      "title": "Secret Name",
      "type": "string"
    },
    "secret_parameters": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret Parameters"
    }
  },
  "required": [
    "data_path",
    "secret_name"
  ],
  "title": "R2StorageConfig",
  "type": "object"
}

key_id `pydantic-field` ¶

key_id: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
key_id = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
key_id = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__KEY_ID=...

secret `pydantic-field` ¶

secret: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET=...

account_id `pydantic-field` ¶

account_id: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
account_id = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
account_id = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__ACCOUNT_ID=...

data_path `pydantic-field` ¶

data_path: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
data_path = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
data_path = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__DATA_PATH=...

secret_name `pydantic-field` ¶

secret_name: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_name = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_name = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_NAME=...

secret_parameters `pydantic-field` ¶

secret_parameters: dict[str, Any] | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_parameters = {}

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_parameters = {}

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_PARAMETERS=...

Google Cloud Storage backend for DuckLake.

Uses the DuckDB TYPE GCS secret.

Show JSON schema:

{
  "description": "Google Cloud Storage backend for [DuckLake](https://ducklake.select/).\n\nUses the DuckDB [`TYPE GCS`](https://duckdb.org/docs/stable/core_extensions/httpfs/s3api#gcs-secrets) secret.",
  "properties": {
    "type": {
      "const": "gcs",
      "default": "gcs",
      "title": "Type",
      "type": "string"
    },
    "key_id": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Key Id"
    },
    "secret": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret"
    },
    "data_path": {
      "title": "Data Path",
      "type": "string"
    },
    "secret_name": {
      "title": "Secret Name",
      "type": "string"
    },
    "secret_parameters": {
      "anyOf": [
        {
          "additionalProperties": true,
          "type": "object"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Secret Parameters"
    }
  },
  "required": [
    "data_path",
    "secret_name"
  ],
  "title": "GCSStorageConfig",
  "type": "object"
}

key_id `pydantic-field` ¶

key_id: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
key_id = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
key_id = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__KEY_ID=...

secret `pydantic-field` ¶

secret: str | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET=...

data_path `pydantic-field` ¶

data_path: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
data_path = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
data_path = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__DATA_PATH=...

secret_name `pydantic-field` ¶

secret_name: str

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_name = "..."

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_name = "..."

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_NAME=...

secret_parameters `pydantic-field` ¶

secret_parameters: dict[str, Any] | None = None

metaxy.tomlpyproject.tomlEnvironment Variable

[stores.dev.config.ducklake.storage]
secret_parameters = {}

[tool.metaxy.stores.dev.config.ducklake.storage]
secret_parameters = {}

export METAXY_STORES__DEV__CONFIG__DUCKLAKE__STORAGE__SECRET_PARAMETERS=...

Metaxy + DuckLake¶

Configuration¶

catalog pydantic-field ¶

storage pydantic-field ¶

alias pydantic-field ¶

attach_options pydantic-field ¶

data_inlining_row_limit pydantic-field ¶

Catalog Backends¶

uri pydantic-field ¶

uri pydantic-field ¶

database pydantic-field ¶

user pydantic-field ¶

password pydantic-field ¶

host pydantic-field ¶

port pydantic-field ¶

secret_name pydantic-field ¶

secret_parameters pydantic-field ¶

database pydantic-field ¶

region pydantic-field ¶

Storage Backends¶

path pydantic-field ¶

key_id pydantic-field ¶

secret pydantic-field ¶

endpoint pydantic-field ¶

bucket pydantic-field ¶

prefix pydantic-field ¶

region pydantic-field ¶

url_style pydantic-field ¶

use_ssl pydantic-field ¶

scope pydantic-field ¶

data_path pydantic-field ¶

secret_name pydantic-field ¶

secret_parameters pydantic-field ¶

key_id pydantic-field ¶

secret pydantic-field ¶

account_id pydantic-field ¶

data_path pydantic-field ¶

secret_name pydantic-field ¶

secret_parameters pydantic-field ¶

key_id pydantic-field ¶

secret pydantic-field ¶

data_path pydantic-field ¶

secret_name pydantic-field ¶

secret_parameters pydantic-field ¶

catalog `pydantic-field` ¶

storage `pydantic-field` ¶

alias `pydantic-field` ¶

attach_options `pydantic-field` ¶

data_inlining_row_limit `pydantic-field` ¶

uri `pydantic-field` ¶

uri `pydantic-field` ¶

database `pydantic-field` ¶

user `pydantic-field` ¶

password `pydantic-field` ¶

host `pydantic-field` ¶

port `pydantic-field` ¶

secret_name `pydantic-field` ¶

secret_parameters `pydantic-field` ¶

database `pydantic-field` ¶

region `pydantic-field` ¶

path `pydantic-field` ¶

key_id `pydantic-field` ¶

secret `pydantic-field` ¶

endpoint `pydantic-field` ¶

bucket `pydantic-field` ¶

prefix `pydantic-field` ¶

region `pydantic-field` ¶

url_style `pydantic-field` ¶

use_ssl `pydantic-field` ¶

scope `pydantic-field` ¶

data_path `pydantic-field` ¶

secret_name `pydantic-field` ¶

secret_parameters `pydantic-field` ¶

key_id `pydantic-field` ¶

secret `pydantic-field` ¶

account_id `pydantic-field` ¶

data_path `pydantic-field` ¶

secret_name `pydantic-field` ¶

secret_parameters `pydantic-field` ¶

key_id `pydantic-field` ¶

secret `pydantic-field` ¶

data_path `pydantic-field` ¶

secret_name `pydantic-field` ¶

secret_parameters `pydantic-field` ¶