diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9249544..c6af086 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -48,6 +48,7 @@ export default defineConfig(withMermaid({ useTitleFromFrontmatter: true, useFolderLinkFromIndexFile: true, useFolderTitleFromIndexFile: true, + sortMenusByFrontmatterOrder: true, }), socialLinks: [ diff --git a/docs/architecture/cloud-storage/arbiter.md b/docs/architecture/cloud-storage/arbiter.md index 5ee1641..f23fd24 100644 --- a/docs/architecture/cloud-storage/arbiter.md +++ b/docs/architecture/cloud-storage/arbiter.md @@ -1,5 +1,6 @@ --- title: Arbiter +order: 60 --- # Arbiter diff --git a/docs/architecture/cloud-storage/ceph.md b/docs/architecture/cloud-storage/ceph.md index 02ce1ce..bec28b1 100644 --- a/docs/architecture/cloud-storage/ceph.md +++ b/docs/architecture/cloud-storage/ceph.md @@ -1,5 +1,6 @@ --- title: Ceph +order: 30 --- # Ceph diff --git a/docs/architecture/cloud-storage/chorus.md b/docs/architecture/cloud-storage/chorus.md index 017cb9f..4682ac7 100644 --- a/docs/architecture/cloud-storage/chorus.md +++ b/docs/architecture/cloud-storage/chorus.md @@ -1,5 +1,6 @@ --- title: Chorus +order: 50 --- # Chorus diff --git a/docs/architecture/cloud-storage/index.md b/docs/architecture/cloud-storage/index.md index 896c7c3..c1c1b89 100644 --- a/docs/architecture/cloud-storage/index.md +++ b/docs/architecture/cloud-storage/index.md @@ -1,5 +1,6 @@ --- title: Cloud Storage +order: 50 --- # Cloud Storage @@ -41,7 +42,7 @@ Ceph exposes three storage interfaces that CobaltCore services consume: Applications / VMs │ ┌───────┴────────────────────┐ -│ RBD │ CephFS │ RGW │ ← Ceph interfaces +│ RBD │ CephFS │ RGW │ ← Ceph interfaces └───────┴────────────────────┘ │ RADOS (Reliable Autonomic Distributed Object Store) diff --git a/docs/architecture/cloud-storage/liquid-ceph.md b/docs/architecture/cloud-storage/liquid-ceph.md index 7b1943f..3c3a43a 100644 --- a/docs/architecture/cloud-storage/liquid-ceph.md +++ b/docs/architecture/cloud-storage/liquid-ceph.md @@ -1,5 +1,6 @@ --- title: Liquid-Ceph +order: 70 --- # Liquid-Ceph diff --git a/docs/architecture/cloud-storage/observability/index.md b/docs/architecture/cloud-storage/observability/index.md index b01cb1a..9656046 100644 --- a/docs/architecture/cloud-storage/observability/index.md +++ b/docs/architecture/cloud-storage/observability/index.md @@ -1,5 +1,6 @@ --- title: Observability & Audit +order: 80 --- # Observability & Audit Overview diff --git a/docs/architecture/cloud-storage/overview.md b/docs/architecture/cloud-storage/overview.md new file mode 100644 index 0000000..8337a8f --- /dev/null +++ b/docs/architecture/cloud-storage/overview.md @@ -0,0 +1,75 @@ +--- +title: Overview - Cloud Storage +order: 15 +--- + +# Overview - Cloud Storage + +CobaltCore's cloud storage layer is built on [Ceph](./ceph.md), a distributed storage system that delivers object, block, and file storage in a single unified platform. The surrounding components handle lifecycle automation, data replication, high-availability quorum, observability, and liquid storage allocation — each with a focused responsibility. + +## Architecture + +The storage stack is organized into three layers: + +**Foundation** — Ceph provides the core distributed storage engine. All other components either operate it, extend it, or observe it. + +**Operations** — [Rook](./rook.md) runs as a Kubernetes operator and manages the full lifecycle of Ceph daemons (monitors, managers, OSDs, MDS, RGW) as containerized workloads. [Arbiter](./arbiter.md) extends quorum into stretched cluster topologies by deploying external Ceph monitors that Rook does not manage directly. + +**Data Services** — [Chorus](./chorus.md) provides zero-downtime data replication and migration between object storage systems (S3 and Swift). [Liquid-Ceph](./liquid-ceph.md) enables dynamic, on-demand storage allocation across the cluster. + +## Components + +| Component | Layer | Role | +|-----------|-------|------| +| [Ceph](./ceph.md) | Foundation | Distributed storage engine — block (RBD), file (CephFS), object (RGW) | +| [Rook](./rook.md) | Operations | Kubernetes operator for Ceph lifecycle management | +| [Arbiter](./arbiter.md) | Operations | External Ceph monitors for quorum in stretched clusters | +| [Chorus](./chorus.md) | Data Services | Zero-downtime object storage replication and migration | +| [Liquid-Ceph](./liquid-ceph.md) | Data Services | Dynamic storage allocation across the Ceph cluster | +| [Observability & Audit](./observability/) | Observability | Metrics, dashboards, alerting, and audit — Prometheus, Perses, Prysm | + +## Storage Interfaces + +Ceph exposes three storage interfaces that CobaltCore services consume: + +- **RBD (RADOS Block Device)** — thin-provisioned, resizable block volumes used by virtual machines and databases. Striped across OSDs for parallel I/O and backed by RADOS snapshots and replication. +- **CephFS** — POSIX-compliant distributed filesystem. Metadata is managed by a dedicated MDS cluster; data is striped across OSDs. Supports snapshots, quotas, and multiple active MDS daemons for horizontal metadata scaling. +- **RGW (RADOS Gateway)** — S3 and Swift-compatible object storage gateway. Supports multi-tenancy, versioning, lifecycle policies, server-side encryption, and multi-site active-active replication. + +## Data Flow + +```text +Applications / VMs + │ +┌───────┴────────────────────┐ +│ RBD │ CephFS │ RGW │ ← Ceph interfaces +└───────┴────────────────────┘ + │ + RADOS (Reliable Autonomic Distributed Object Store) + │ + OSDs across cluster nodes + │ + ┌────┴─────┐ + │ Rook │ ← manages daemon lifecycle via Kubernetes CRDs + └──────────┘ + │ + ┌────┴──────┐ ┌─────────┐ ┌────────────┐ + │ Arbiter │ │ Chorus │ │ Liquid-Ceph│ + └───────────┘ └─────────┘ └────────────┘ + (quorum) (replication) (allocation) + │ + ┌────┴──────────────────────────┐ + │ Observability & Audit │ + │ Prometheus · Perses · Prysm │ + └───────────────────────────────┘ +``` + +## High Availability + +Ceph achieves HA through monitor quorum (typically 3 or 5 monitors), OSD replication or erasure coding, and MDS standby daemons. In stretched deployments that span two sites, [Arbiter](./arbiter.md) deploys a third monitor at a tiebreaker site so that quorum is maintained even if one full site goes offline. + +## See Also + +- [Observability & Audit](./observability/) — Prometheus metrics, Perses dashboards, and Prysm CLI for the storage stack +- [Ceph upstream architecture docs](https://docs.ceph.com/en/latest/architecture/) +- [Rook documentation](https://rook.io/docs/rook/latest-release/Getting-Started/intro/) diff --git a/docs/architecture/cloud-storage/rook-ceph.md b/docs/architecture/cloud-storage/rook-ceph.md index 4a7ce23..c62656b 100644 --- a/docs/architecture/cloud-storage/rook-ceph.md +++ b/docs/architecture/cloud-storage/rook-ceph.md @@ -1,5 +1,6 @@ --- title: Installing Rook-Ceph on Kubernetes +order: 25 --- # Installing Rook-Ceph on Kubernetes diff --git a/docs/architecture/cloud-storage/rook.md b/docs/architecture/cloud-storage/rook.md index b803e14..bccc257 100644 --- a/docs/architecture/cloud-storage/rook.md +++ b/docs/architecture/cloud-storage/rook.md @@ -1,5 +1,6 @@ --- title: Rook +order: 20 --- # Rook diff --git a/docs/architecture/cluster.md b/docs/architecture/cluster.md index 4f9c34b..afe64aa 100644 --- a/docs/architecture/cluster.md +++ b/docs/architecture/cluster.md @@ -1,5 +1,6 @@ --- title: Kubernetes Cluster +order: 10 --- # Kubernetes Cluster diff --git a/docs/architecture/cortex.md b/docs/architecture/cortex.md index 5222d0c..8d0ee79 100644 --- a/docs/architecture/cortex.md +++ b/docs/architecture/cortex.md @@ -1,5 +1,6 @@ --- title: Cortex +order: 40 --- # Cortex @@ -66,4 +67,4 @@ graph LR External <-->|receive placement request\nreturn placement suggestion| IniPlacement External <-->|trigger migration| Scheduler API <--> User["User / Dashboard"] -``` \ No newline at end of file +``` diff --git a/docs/architecture/hypervisor.md b/docs/architecture/hypervisor.md index 9152c8f..8493860 100644 --- a/docs/architecture/hypervisor.md +++ b/docs/architecture/hypervisor.md @@ -1,5 +1,6 @@ --- title: Hypervisor +order: 30 --- # Hypervisor @@ -168,4 +169,4 @@ NAME ORIGIN DESTINATION TYPE 12e479eb-6bef-4fdb-bfdc-0388df68bed9 node002-bb086 node008-bb086 completed migration_in 74d 2.755s 4.0 GiB 10.4 MiB 0 B 16.5 MiB/s 0/s 3 13679335-291c-4405-9e08-5911032599dd node007-bb086 node009-bb086 completed migration_out 78d 2.766s 4.0 GiB 398.0 MiB 718.8 MiB 588.9 MiB/s 0/s 1 1552e60a-bdba-4850-84da-07dd635bce2c node006-bb086 node003-bb087 completed migration_out 22d 35.078s 64.0 GiB 58.4 GiB 0 B 1.8 GiB/s 1250/s 4 -``` \ No newline at end of file +``` diff --git a/docs/architecture/openstack.md b/docs/architecture/openstack.md index 7f96526..ee4a4a3 100644 --- a/docs/architecture/openstack.md +++ b/docs/architecture/openstack.md @@ -1,5 +1,6 @@ --- title: OpenStack +order: 20 --- # OpenStack @@ -141,4 +142,4 @@ C4Deployment Rel(computeagent, cinderapi, "Calls") Rel(mount, netapp, "Accesses", "NFS") UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1") -``` \ No newline at end of file +``` diff --git a/docs/architecture/rook-ceph-install.md b/docs/architecture/rook-ceph-install.md deleted file mode 100644 index 532ccef..0000000 --- a/docs/architecture/rook-ceph-install.md +++ /dev/null @@ -1,432 +0,0 @@ ---- -title: Installing Rook-Ceph on Kubernetes ---- - -# Installing Rook-Ceph on Kubernetes - -## Overview - -This guide provides step-by-step instructions for deploying a Ceph storage -cluster using the Rook operator on Kubernetes. Rook automates the deployment, -configuration, and management of Ceph clusters within Kubernetes environments. - -The instructions here are meant only as a general guideline. We recommend that -you use the instructions found in the [official Rook -documentation](https://rook.io/docs/rook/latest/) and the [upstream Ceph -documentation](https://docs.ceph.com/). - - -## Prerequisites - -Before beginning the installation, ensure the following requirements are met: - -### Kubernetes Cluster Requirements - -- Kubernetes v1.25 or higher -- `kubectl` configured to communicate with your cluster -- Administrator access to the Kubernetes cluster -- At least 3 worker nodes for a production cluster (1 node minimum for testing) -- Verify compatibility between your Kubernetes version and the Rook version you -  intend to deploy — see the [Rook releases page](https://github.com/rook/rook/releases) -  for version compatibility information - -### Storage Requirements - -- Raw block devices available on worker nodes (unformatted, no filesystem) -- Minimum 10 GB of storage per OSD -- Devices should not be mounted or in use by the operating system - -### Network Requirements - -- Network connectivity between all cluster nodes -- Network access between pods is handled by the Kubernetes network plugin (CNI). -  Ensure your CNI supports the required pod-to-pod communication. If you need -  to open ports for external access to Ceph services, the typical ports are -  6789, 3300, and 6800-7300. - -### System Requirements - -- Linux kernel 4.5 or higher (5.x recommended) -- LVM2 packages installed on all nodes -- Minimum 2 GB RAM per node (4 GB+ recommended) -- `helm` installed if using Helm-based deployment (optional) - -## Installation Steps - -### Step 1: Clone the Rook Repository - -Clone the Rook repository to get the deployment manifests: - -```bash -git clone --single-branch --branch release- https://github.com/rook/rook.git -cd rook/deploy/examples -``` - -**Note:** Replace `release-` with the desired Rook version. Check the Rook -releases page for the latest stable version, and verify it is compatible with -your Kubernetes version before proceeding. - -### Step 2: Deploy the Rook Operator - -Install the Rook operator, which manages the Ceph cluster lifecycle: - -```bash -# Deploy common resources -kubectl create -f crds.yaml -kubectl create -f common.yaml - -# Deploy the Rook operator -kubectl create -f operator.yaml -``` - -### Step 3: Verify Operator Deployment - -Confirm the Rook operator is running: - -```bash -kubectl -n rook-ceph get pods -``` - -**Expected output:** - -``` -NAME                                  READY   STATUS    RESTARTS   AGE -rook-ceph-operator-           1/1     Running   0          30s -``` - -Wait until the operator pod shows `Running` status before proceeding. - -### Step 4: Create the Ceph Cluster - -Deploy a Ceph cluster using the cluster manifest: - -```bash -kubectl create -f cluster.yaml -``` - -This creates a basic Ceph cluster with the following default configuration: - -- 3 Mon (Monitor) daemons -- 1 Mgr (Manager) daemon -- OSDs automatically provisioned from available devices -- Dashboard enabled - -### Step 5: Monitor Cluster Deployment - -Watch the cluster deployment progress: - -```bash -kubectl -n rook-ceph get pods -w -``` - -The deployment is complete when all pods are in `Running` state. This may take -several minutes. - -**Expected pods:** - -- `rook-ceph-mon-*` - Monitor daemons (typically 3) -- `rook-ceph-mgr-*` - Manager daemons (typically 2) -- `rook-ceph-osd-*` - OSD daemons (one per device) -- `rook-ceph-crashcollector-*` - Crash collectors (one per node) - -### Step 6: Verify Cluster Health - -Check the Ceph cluster health: - -```bash -# Deploy the Rook toolbox for cluster management -kubectl create -f toolbox.yaml - -# Wait for toolbox to be ready -kubectl -n rook-ceph rollout status deployment/rook-ceph-tools - -# Check cluster status -kubectl -n rook-ceph exec -it deployment/rook-ceph-tools -- ceph status -``` - -**Healthy cluster output should show:** - -``` -cluster: -  id:     -  health: HEALTH_OK - -services: -  mon: 3 daemons, quorum a,b,c -  mgr: a(active), standbys: b -  osd: X osds: X up, X in -``` - -## Configuration Options - -### Customizing the Cluster - -Edit `cluster.yaml` to customize your deployment before creating the cluster: - -#### Storage Configuration - -Specify which devices to use for OSDs: - -```yaml -storage: -  useAllNodes: true -  useAllDevices: false -  deviceFilter: "^sd[b-z]"  # Use sdb, sdc, etc. -``` - -Or specify devices explicitly: - -```yaml -storage: -  nodes: -  - name: "node1" -    devices: -    - name: "/dev/sdb" -  - name: "node2" -    devices: -    - name: "/dev/sdc" -``` - -#### Resource Limits - -Set resource limits for Ceph daemons: - -```yaml -resources: -  mon: -    limits: -      cpu: "2000m" -      memory: "4Gi" -    requests: -      cpu: "1000m" -      memory: "2Gi" -  osd: -    limits: -      cpu: "2000m" -      memory: "4Gi" -    requests: -      cpu: "1000m" -      memory: "2Gi" -``` - -#### Network Configuration - -Configure network settings for client and cluster traffic: - -```yaml -network: -  provider: host  # or multus for advanced networking -  # Uncomment for dual network configuration -  # connections: -  #   encryption: -  #     enabled: true -``` - -### Dashboard Access - -Enable and access the Ceph dashboard: - -```bash -# The dashboard is enabled by default in cluster.yaml - -# Get the dashboard password -kubectl -n rook-ceph get secret rook-ceph-dashboard-password \ -  -o jsonpath="{['data']['password']}" | base64 --decode && echo - -# Port-forward to access the dashboard -kubectl -n rook-ceph port-forward service/rook-ceph-mgr-dashboard 8443:8443 -``` - -Access the dashboard at: `https://localhost:8443` - -Username: `admin` -Password: (from the command above) - -## Creating Storage Classes - -### Block Storage (RBD) - -Create a storage class for block devices: - -```bash -kubectl create -f csi/rbd/storageclass.yaml -``` - -Test the storage class: - -```bash -# Create a test PVC -cat <