Feature/external catalog config

config/external_config.json

@@ -0,0 +1,171 @@
+{
+  "id": "dbt-common/external-catalog-config-v0",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "..",
+  "title": "External Catalog Config",
+  "type": "object",
+  "required": [
+    "catalogs"
+  ],
+  "properties": {
+        "catalogs": {
+          "type": "array",
+          "items": {
+                "$ref": "#/$defs/externalCatalog"
+            }
+        }
+    },
+  "$defs": {
+    "icebergConfiguration": {
+      "type": "object",
+      "required": [
+        "table_format",
+        "external_location",
+        "internal_namespace",
+        "location"
+      ],
+      "properties": {
+        "table_format": {
+            "type": "string",
+            "description": "The table format",
+            "default": "iceberg"
+            },
+        "catalog_namespace": {
+          "type": "string",
+          "description": "The namespace",
+          "default": "dbt"
+        },
+        "internal_namespace": {
+          "type": "object",
+            "required": [
+                "database",
+                "schema"
+            ],
+            "properties": {
+                "database": {
+                "type": "string",
+                "description": "The database"
+                },
+                "schema": {
+                "type": "string",
+                "description": "The schema"
+                }
+            }
+        },
+        "external_volume": {
+          "type": "object",
+          "required": [
+                    "type",
+                    "path"
+                ],
+          "properties": {
+              "external_location": {
+                "type": "string",
+                "description": "The external location URI",
+                "format": "uri"
+              },
+              "name": {
+              "type": "string",
+              "description": "The name of the volume"
+              }
+            }
+          }
+        }
+      },
+    "glueConfiguration": {
+      "type": "object",
+      "required": [
+        "table_format",
+        "external_location",
+        "location"
+      ],
+      "properties": {
+        "table_format": {
+            "type": "string",
+            "description": "The table format"
+            },
+        "namespace": {
+          "type": "string",
+          "description": "The namespace",
+          "default": "dbt"
+        },
+        "external_location": {
+          "type": "string",
+          "description": "The external location",
+          "format": "uri"
+        },
+        "aws_account_id": {
+          "type": "string",
+          "description": "The AWS account ID"
+        },
+        "role_arn": {
+          "type": "string",
+          "description": "The role ARN"
+        }
+        }
+      },
+    "management": {
+      "type": "object",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "Whether management is enabled"
+        },
+        "create_if_not_exists": {
+          "type": "boolean",
+          "default": false,
+          "description": "Whether to create the external catalog if it does not exist"
+        },
+        "alter_if_different": {
+          "type": "boolean",
+          "default": false,
+          "description": "Whether to alter the external catalog if it exists"
+        },
+        "read_only": {
+          "type": "boolean",
+          "default": true,
+          "description": "Whether the external catalog is read-only"
+        },
+        "refresh": {
+          "type": "string",
+          "enum": [
+            "on-start",
+            "never",
+            "just-in-time"
+          ],
+          "default": "on-start",
+          "description": "Whether to refresh the external catalog"
+        }
+      }
+    },
+    "externalCatalog": {
+      "type": "object",
+      "required": [
+        "type",
+        "name",
+        "configuration",
+        "management"
+      ],
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "The name of the external catalog"
+        },
+        "type": {
+          "enum": [
+            "iceberg",
+            "glue"
+          ]
+        },
+        "configuration": {
+          "type": "object",
+          "oneOf": [{"$ref": "#/$defs/icebergConfiguration"}, {"$ref": "#/$defs/glueConfiguration"}]
+        },
+        "management": {
+          "$ref": "#/$defs/management"
+        }
+      }
+  }
+}
+}

config/pyproject.toml

@@ -0,0 +1,172 @@
+[project]
+name = "dbt-config"
+dynamic = ["version"]
+description = "The shared configuration interfaces"
+readme = "README.md"
+requires-python = ">=3.8"
+license = "Apache-2.0"
+keywords = []
+authors = [
+  { name = "dbt Labs", email = "[email protected]" },
+]
+maintainers = [
+    { name = "dbt Labs", email = "[email protected]" },
+]
+classifiers = [
+  "Development Status :: 2 - Pre-Alpha",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: MacOS :: MacOS X",
+  "Operating System :: Microsoft :: Windows",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = [
+  "pydantic"
+]
+
+[project.optional-dependencies]
+lint = [
+    "black>=23.3,<24.0",
+    "flake8",
+    "flake8-pyproject",
+    "flake8-docstrings",
+    "mypy>=1.3,<2.0",
+    "pytest>=7.3,<8.0",  # needed for linting tests
+    "types-Jinja2>=2.11,<3.0",
+    "types-jsonschema>=4.17,<5.0",
+    "types-protobuf>=4.24,<5.0",
+    "types-python-dateutil>=2.8,<3.0",
+    "types-PyYAML>=6.0,<7.0",
+    "types-requests"
+]
+test = [
+    "pytest>=7.3,<8.0",
+    "pytest-mock",
+    "pytest-xdist>=3.2,<4.0",
+    "pytest-cov>=4.1,<5.0",
+    "hypothesis>=6.87,<7.0",
+]
+build = [
+    "wheel",
+    "twine",
+    "check-wheel-contents",
+]
+
+[project.urls]
+Homepage = "https://github.com/dbt-labs/dbt-common/config/"
+Repository = "https://github.com/dbt-labs/dbt-common.git"
+Issues = "https://github.com/dbt-labs/dbt-common/issues"
+Changelog = "https://github.com/dbt-labs/dbt-common/blob/main/CHANGELOG.md"
+
+[tool.hatch.version]
+path = "src/dbt_config/__about__.py"
+
+### Default env & scripts
+
+[tool.hatch.envs.default]
+description = "Default environment with dependencies for running dbt-config"
+features = ["lint", "test"]
+
+### Test settings, envs & scripts
+[tool.hatch.envs.test]
+description = "Env for running development commands for testing"
+features = ["test"]
+
+[tool.hatch.envs.test.scripts]
+unit = "python -m pytest --cov=dbt_config --cov-report=xml test/"
+
+### Linting settings, envs & scripts
+
+[tool.hatch.envs.lint]
+type = "virtual"
+description = "Env for running development commands for linting"
+features = ["lint"]
+
+[tool.hatch.envs.lint.scripts]
+all = [
+    "- black",
+    "- flake8",
+    "- mypy",
+]
+black = "python -m black ."
+flake8 = "python -m flake8 ."
+mypy = "python -m mypy ."
+
+[tool.black]
+line-length = 99
+target-version = ['py38']
+
+[tool.flake8]
+max-line-length = 99
+select = ["E", "W", "F"]
+ignore = ["E203", "E501", "E741", "W503", "W504"]
+exclude = [
+    "venv",
+    ".venv",
+    "env*",
+    ".hatch/*",
+]
+per-file-ignores = ["*/__init__.py: F401", "*/conftest.py: F401"]
+docstring-convention = "google"
+
+[tool.mypy]
+mypy_path = "third-party-stubs/"
+namespace_packages = true
+warn_unused_configs = true
+show_error_codes = true
+disable_error_code = "attr-defined"  # TODO: revisit once other mypy errors resolved
+disallow_untyped_defs = false # TODO: add type annotations everywhere
+warn_redundant_casts = true
+ignore_missing_imports = true
+exclude = [
+    "env*",
+    "third-party-stubs/*",
+]
+
+
+### Build settings, envs & scripts
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+exclude = [
+  "/.github",
+  "/.changes",
+  ".changie.yaml",
+  ".gitignore",
+  ".pre-commit-config.yaml",
+  "CONTRIBUTING.md",
+  "/test",
+]
+
+[tool.hatch.build.targets.wheel]
+
+[tool.hatch.envs.build]
+description = "Env for running development commands for linting"
+features = ["build"]
+packages = ["src/dbt_config"]
+
+[tool.hatch.envs.build.scripts]
+check-all = [
+    "- check-wheel",
+    "- check-sdist",
+]
+check-wheel = [
+    "twine check dist/*",
+    "find ./dist/dbt_config-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
+    "pip freeze | grep dbt-config",
+]
+check-sdist = [
+    "check-wheel-contents dist/*.whl --ignore W007,W008",
+    "find ./dist/dbt_config-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
+    "pip freeze | grep dbt-config",
+]

config/src/dbt_config/__about__.py

@@ -0,0 +1 @@
+version = "0.0.1"