From 628439ce4fa4bf687f552f9a4ae41f2da901be71 Mon Sep 17 00:00:00 2001 From: Zac Dover Date: Mon, 13 Jul 2026 15:15:51 +1000 Subject: [PATCH 1/2] doc: fix vitepress sidebar (option 1 of 2) Order the pages in the sidebar so that they are (nearly) congruent with the schema agreed upon between Barbara and SAP. This is the first of two options I am creating on 13 Jul 2026. In this option, the Prysm page remains in the "Observability" folder. This is the location in which I found the Prysm page when I rearranged the other pages. It is reasonable to group the Prysm page under the rubric of "Observability", so in this commit I am leaving it where it is. The second option that I will create today will move the Prysm page between the Ceph page and the Chorus page in the "Cloud Storage" hierarchy. The second option is absolutely congruent with the schema agreed upon between Barbara and SAP. Either this commit (the first option) or the other commit (the second option) should be merged. The commit that is not merged should be rejected. Both options cannot be merged without creating a merge conflict. Signed-off-by: Zac Dover --- docs/.vitepress/config.mts | 1 + docs/architecture/cloud-storage/arbiter.md | 1 + docs/architecture/cloud-storage/ceph.md | 1 + docs/architecture/cloud-storage/chorus.md | 1 + docs/architecture/cloud-storage/index.md | 3 +- .../architecture/cloud-storage/liquid-ceph.md | 1 + .../cloud-storage/observability/index.md | 1 + docs/architecture/cloud-storage/overview.md | 75 +++ docs/architecture/cloud-storage/rook-ceph.md | 1 + docs/architecture/cloud-storage/rook.md | 1 + docs/architecture/cluster.md | 1 + docs/architecture/cortex.md | 3 +- docs/architecture/hypervisor.md | 3 +- docs/architecture/openstack.md | 3 +- docs/architecture/rook-ceph-install.md | 432 ------------------ 15 files changed, 92 insertions(+), 436 deletions(-) create mode 100644 docs/architecture/cloud-storage/overview.md delete mode 100644 docs/architecture/rook-ceph-install.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 9249544..c6af086 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -48,6 +48,7 @@ export default defineConfig(withMermaid({ useTitleFromFrontmatter: true, useFolderLinkFromIndexFile: true, useFolderTitleFromIndexFile: true, + sortMenusByFrontmatterOrder: true, }), socialLinks: [ diff --git a/docs/architecture/cloud-storage/arbiter.md b/docs/architecture/cloud-storage/arbiter.md index 5ee1641..f23fd24 100644 --- a/docs/architecture/cloud-storage/arbiter.md +++ b/docs/architecture/cloud-storage/arbiter.md @@ -1,5 +1,6 @@ --- title: Arbiter +order: 60 --- # Arbiter diff --git a/docs/architecture/cloud-storage/ceph.md b/docs/architecture/cloud-storage/ceph.md index 02ce1ce..bec28b1 100644 --- a/docs/architecture/cloud-storage/ceph.md +++ b/docs/architecture/cloud-storage/ceph.md @@ -1,5 +1,6 @@ --- title: Ceph +order: 30 --- # Ceph diff --git a/docs/architecture/cloud-storage/chorus.md b/docs/architecture/cloud-storage/chorus.md index 017cb9f..4682ac7 100644 --- a/docs/architecture/cloud-storage/chorus.md +++ b/docs/architecture/cloud-storage/chorus.md @@ -1,5 +1,6 @@ --- title: Chorus +order: 50 --- # Chorus diff --git a/docs/architecture/cloud-storage/index.md b/docs/architecture/cloud-storage/index.md index 896c7c3..c1c1b89 100644 --- a/docs/architecture/cloud-storage/index.md +++ b/docs/architecture/cloud-storage/index.md @@ -1,5 +1,6 @@ --- title: Cloud Storage +order: 50 --- # Cloud Storage @@ -41,7 +42,7 @@ Ceph exposes three storage interfaces that CobaltCore services consume: Applications / VMs │ ┌───────┴────────────────────┐ -│ RBD │ CephFS │ RGW │ ← Ceph interfaces +│ RBD │ CephFS │ RGW │ ← Ceph interfaces └───────┴────────────────────┘ │ RADOS (Reliable Autonomic Distributed Object Store) diff --git a/docs/architecture/cloud-storage/liquid-ceph.md b/docs/architecture/cloud-storage/liquid-ceph.md index 7b1943f..3c3a43a 100644 --- a/docs/architecture/cloud-storage/liquid-ceph.md +++ b/docs/architecture/cloud-storage/liquid-ceph.md @@ -1,5 +1,6 @@ --- title: Liquid-Ceph +order: 70 --- # Liquid-Ceph diff --git a/docs/architecture/cloud-storage/observability/index.md b/docs/architecture/cloud-storage/observability/index.md index b01cb1a..9656046 100644 --- a/docs/architecture/cloud-storage/observability/index.md +++ b/docs/architecture/cloud-storage/observability/index.md @@ -1,5 +1,6 @@ --- title: Observability & Audit +order: 80 --- # Observability & Audit Overview diff --git a/docs/architecture/cloud-storage/overview.md b/docs/architecture/cloud-storage/overview.md new file mode 100644 index 0000000..8337a8f --- /dev/null +++ b/docs/architecture/cloud-storage/overview.md @@ -0,0 +1,75 @@ +--- +title: Overview - Cloud Storage +order: 15 +--- + +# Overview - Cloud Storage + +CobaltCore's cloud storage layer is built on [Ceph](./ceph.md), a distributed storage system that delivers object, block, and file storage in a single unified platform. The surrounding components handle lifecycle automation, data replication, high-availability quorum, observability, and liquid storage allocation — each with a focused responsibility. + +## Architecture + +The storage stack is organized into three layers: + +**Foundation** — Ceph provides the core distributed storage engine. All other components either operate it, extend it, or observe it. + +**Operations** — [Rook](./rook.md) runs as a Kubernetes operator and manages the full lifecycle of Ceph daemons (monitors, managers, OSDs, MDS, RGW) as containerized workloads. [Arbiter](./arbiter.md) extends quorum into stretched cluster topologies by deploying external Ceph monitors that Rook does not manage directly. + +**Data Services** — [Chorus](./chorus.md) provides zero-downtime data replication and migration between object storage systems (S3 and Swift). [Liquid-Ceph](./liquid-ceph.md) enables dynamic, on-demand storage allocation across the cluster. + +## Components + +| Component | Layer | Role | +|-----------|-------|------| +| [Ceph](./ceph.md) | Foundation | Distributed storage engine — block (RBD), file (CephFS), object (RGW) | +| [Rook](./rook.md) | Operations | Kubernetes operator for Ceph lifecycle management | +| [Arbiter](./arbiter.md) | Operations | External Ceph monitors for quorum in stretched clusters | +| [Chorus](./chorus.md) | Data Services | Zero-downtime object storage replication and migration | +| [Liquid-Ceph](./liquid-ceph.md) | Data Services | Dynamic storage allocation across the Ceph cluster | +| [Observability & Audit](./observability/) | Observability | Metrics, dashboards, alerting, and audit — Prometheus, Perses, Prysm | + +## Storage Interfaces + +Ceph exposes three storage interfaces that CobaltCore services consume: + +- **RBD (RADOS Block Device)** — thin-provisioned, resizable block volumes used by virtual machines and databases. Striped across OSDs for parallel I/O and backed by RADOS snapshots and replication. +- **CephFS** — POSIX-compliant distributed filesystem. Metadata is managed by a dedicated MDS cluster; data is striped across OSDs. Supports snapshots, quotas, and multiple active MDS daemons for horizontal metadata scaling. +- **RGW (RADOS Gateway)** — S3 and Swift-compatible object storage gateway. Supports multi-tenancy, versioning, lifecycle policies, server-side encryption, and multi-site active-active replication. + +## Data Flow + +```text +Applications / VMs + │ +┌───────┴────────────────────┐ +│ RBD │ CephFS │ RGW │ ← Ceph interfaces +└───────┴────────────────────┘ + │ + RADOS (Reliable Autonomic Distributed Object Store) + │ + OSDs across cluster nodes + │ + ┌────┴─────┐ + │ Rook │ ← manages daemon lifecycle via Kubernetes CRDs + └──────────┘ + │ + ┌────┴──────┐ ┌─────────┐ ┌────────────┐ + │ Arbiter │ │ Chorus │ │ Liquid-Ceph│ + └───────────┘ └─────────┘ └────────────┘ + (quorum) (replication) (allocation) + │ + ┌────┴──────────────────────────┐ + │ Observability & Audit │ + │ Prometheus · Perses · Prysm │ + └───────────────────────────────┘ +``` + +## High Availability + +Ceph achieves HA through monitor quorum (typically 3 or 5 monitors), OSD replication or erasure coding, and MDS standby daemons. In stretched deployments that span two sites, [Arbiter](./arbiter.md) deploys a third monitor at a tiebreaker site so that quorum is maintained even if one full site goes offline. + +## See Also + +- [Observability & Audit](./observability/) — Prometheus metrics, Perses dashboards, and Prysm CLI for the storage stack +- [Ceph upstream architecture docs](https://docs.ceph.com/en/latest/architecture/) +- [Rook documentation](https://rook.io/docs/rook/latest-release/Getting-Started/intro/) diff --git a/docs/architecture/cloud-storage/rook-ceph.md b/docs/architecture/cloud-storage/rook-ceph.md index 4a7ce23..c62656b 100644 --- a/docs/architecture/cloud-storage/rook-ceph.md +++ b/docs/architecture/cloud-storage/rook-ceph.md @@ -1,5 +1,6 @@ --- title: Installing Rook-Ceph on Kubernetes +order: 25 --- # Installing Rook-Ceph on Kubernetes diff --git a/docs/architecture/cloud-storage/rook.md b/docs/architecture/cloud-storage/rook.md index b803e14..bccc257 100644 --- a/docs/architecture/cloud-storage/rook.md +++ b/docs/architecture/cloud-storage/rook.md @@ -1,5 +1,6 @@ --- title: Rook +order: 20 --- # Rook diff --git a/docs/architecture/cluster.md b/docs/architecture/cluster.md index 4f9c34b..afe64aa 100644 --- a/docs/architecture/cluster.md +++ b/docs/architecture/cluster.md @@ -1,5 +1,6 @@ --- title: Kubernetes Cluster +order: 10 --- # Kubernetes Cluster diff --git a/docs/architecture/cortex.md b/docs/architecture/cortex.md index 5222d0c..8d0ee79 100644 --- a/docs/architecture/cortex.md +++ b/docs/architecture/cortex.md @@ -1,5 +1,6 @@ --- title: Cortex +order: 40 --- # Cortex @@ -66,4 +67,4 @@ graph LR External <-->|receive placement request\nreturn placement suggestion| IniPlacement External <-->|trigger migration| Scheduler API <--> User["User / Dashboard"] -``` \ No newline at end of file +``` diff --git a/docs/architecture/hypervisor.md b/docs/architecture/hypervisor.md index 9152c8f..8493860 100644 --- a/docs/architecture/hypervisor.md +++ b/docs/architecture/hypervisor.md @@ -1,5 +1,6 @@ --- title: Hypervisor +order: 30 --- # Hypervisor @@ -168,4 +169,4 @@ NAME ORIGIN DESTINATION TYPE 12e479eb-6bef-4fdb-bfdc-0388df68bed9 node002-bb086 node008-bb086 completed migration_in 74d 2.755s 4.0 GiB 10.4 MiB 0 B 16.5 MiB/s 0/s 3 13679335-291c-4405-9e08-5911032599dd node007-bb086 node009-bb086 completed migration_out 78d 2.766s 4.0 GiB 398.0 MiB 718.8 MiB 588.9 MiB/s 0/s 1 1552e60a-bdba-4850-84da-07dd635bce2c node006-bb086 node003-bb087 completed migration_out 22d 35.078s 64.0 GiB 58.4 GiB 0 B 1.8 GiB/s 1250/s 4 -``` \ No newline at end of file +``` diff --git a/docs/architecture/openstack.md b/docs/architecture/openstack.md index 7f96526..ee4a4a3 100644 --- a/docs/architecture/openstack.md +++ b/docs/architecture/openstack.md @@ -1,5 +1,6 @@ --- title: OpenStack +order: 20 --- # OpenStack @@ -141,4 +142,4 @@ C4Deployment Rel(computeagent, cinderapi, "Calls") Rel(mount, netapp, "Accesses", "NFS") UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1") -``` \ No newline at end of file +``` diff --git a/docs/architecture/rook-ceph-install.md b/docs/architecture/rook-ceph-install.md deleted file mode 100644 index 532ccef..0000000 --- a/docs/architecture/rook-ceph-install.md +++ /dev/null @@ -1,432 +0,0 @@ ---- -title: Installing Rook-Ceph on Kubernetes ---- - -# Installing Rook-Ceph on Kubernetes - -## Overview - -This guide provides step-by-step instructions for deploying a Ceph storage -cluster using the Rook operator on Kubernetes. Rook automates the deployment, -configuration, and management of Ceph clusters within Kubernetes environments. - -The instructions here are meant only as a general guideline. We recommend that -you use the instructions found in the [official Rook -documentation](https://rook.io/docs/rook/latest/) and the [upstream Ceph -documentation](https://docs.ceph.com/). - - -## Prerequisites - -Before beginning the installation, ensure the following requirements are met: - -### Kubernetes Cluster Requirements - -- Kubernetes v1.25 or higher -- `kubectl` configured to communicate with your cluster -- Administrator access to the Kubernetes cluster -- At least 3 worker nodes for a production cluster (1 node minimum for testing) -- Verify compatibility between your Kubernetes version and the Rook version you -  intend to deploy — see the [Rook releases page](https://github.com/rook/rook/releases) -  for version compatibility information - -### Storage Requirements - -- Raw block devices available on worker nodes (unformatted, no filesystem) -- Minimum 10 GB of storage per OSD -- Devices should not be mounted or in use by the operating system - -### Network Requirements - -- Network connectivity between all cluster nodes -- Network access between pods is handled by the Kubernetes network plugin (CNI). -  Ensure your CNI supports the required pod-to-pod communication. If you need -  to open ports for external access to Ceph services, the typical ports are -  6789, 3300, and 6800-7300. - -### System Requirements - -- Linux kernel 4.5 or higher (5.x recommended) -- LVM2 packages installed on all nodes -- Minimum 2 GB RAM per node (4 GB+ recommended) -- `helm` installed if using Helm-based deployment (optional) - -## Installation Steps - -### Step 1: Clone the Rook Repository - -Clone the Rook repository to get the deployment manifests: - -```bash -git clone --single-branch --branch release- https://github.com/rook/rook.git -cd rook/deploy/examples -``` - -**Note:** Replace `release-` with the desired Rook version. Check the Rook -releases page for the latest stable version, and verify it is compatible with -your Kubernetes version before proceeding. - -### Step 2: Deploy the Rook Operator - -Install the Rook operator, which manages the Ceph cluster lifecycle: - -```bash -# Deploy common resources -kubectl create -f crds.yaml -kubectl create -f common.yaml - -# Deploy the Rook operator -kubectl create -f operator.yaml -``` - -### Step 3: Verify Operator Deployment - -Confirm the Rook operator is running: - -```bash -kubectl -n rook-ceph get pods -``` - -**Expected output:** - -``` -NAME                                  READY   STATUS    RESTARTS   AGE -rook-ceph-operator-           1/1     Running   0          30s -``` - -Wait until the operator pod shows `Running` status before proceeding. - -### Step 4: Create the Ceph Cluster - -Deploy a Ceph cluster using the cluster manifest: - -```bash -kubectl create -f cluster.yaml -``` - -This creates a basic Ceph cluster with the following default configuration: - -- 3 Mon (Monitor) daemons -- 1 Mgr (Manager) daemon -- OSDs automatically provisioned from available devices -- Dashboard enabled - -### Step 5: Monitor Cluster Deployment - -Watch the cluster deployment progress: - -```bash -kubectl -n rook-ceph get pods -w -``` - -The deployment is complete when all pods are in `Running` state. This may take -several minutes. - -**Expected pods:** - -- `rook-ceph-mon-*` - Monitor daemons (typically 3) -- `rook-ceph-mgr-*` - Manager daemons (typically 2) -- `rook-ceph-osd-*` - OSD daemons (one per device) -- `rook-ceph-crashcollector-*` - Crash collectors (one per node) - -### Step 6: Verify Cluster Health - -Check the Ceph cluster health: - -```bash -# Deploy the Rook toolbox for cluster management -kubectl create -f toolbox.yaml - -# Wait for toolbox to be ready -kubectl -n rook-ceph rollout status deployment/rook-ceph-tools - -# Check cluster status -kubectl -n rook-ceph exec -it deployment/rook-ceph-tools -- ceph status -``` - -**Healthy cluster output should show:** - -``` -cluster: -  id:     -  health: HEALTH_OK - -services: -  mon: 3 daemons, quorum a,b,c -  mgr: a(active), standbys: b -  osd: X osds: X up, X in -``` - -## Configuration Options - -### Customizing the Cluster - -Edit `cluster.yaml` to customize your deployment before creating the cluster: - -#### Storage Configuration - -Specify which devices to use for OSDs: - -```yaml -storage: -  useAllNodes: true -  useAllDevices: false -  deviceFilter: "^sd[b-z]"  # Use sdb, sdc, etc. -``` - -Or specify devices explicitly: - -```yaml -storage: -  nodes: -  - name: "node1" -    devices: -    - name: "/dev/sdb" -  - name: "node2" -    devices: -    - name: "/dev/sdc" -``` - -#### Resource Limits - -Set resource limits for Ceph daemons: - -```yaml -resources: -  mon: -    limits: -      cpu: "2000m" -      memory: "4Gi" -    requests: -      cpu: "1000m" -      memory: "2Gi" -  osd: -    limits: -      cpu: "2000m" -      memory: "4Gi" -    requests: -      cpu: "1000m" -      memory: "2Gi" -``` - -#### Network Configuration - -Configure network settings for client and cluster traffic: - -```yaml -network: -  provider: host  # or multus for advanced networking -  # Uncomment for dual network configuration -  # connections: -  #   encryption: -  #     enabled: true -``` - -### Dashboard Access - -Enable and access the Ceph dashboard: - -```bash -# The dashboard is enabled by default in cluster.yaml - -# Get the dashboard password -kubectl -n rook-ceph get secret rook-ceph-dashboard-password \ -  -o jsonpath="{['data']['password']}" | base64 --decode && echo - -# Port-forward to access the dashboard -kubectl -n rook-ceph port-forward service/rook-ceph-mgr-dashboard 8443:8443 -``` - -Access the dashboard at: `https://localhost:8443` - -Username: `admin` -Password: (from the command above) - -## Creating Storage Classes - -### Block Storage (RBD) - -Create a storage class for block devices: - -```bash -kubectl create -f csi/rbd/storageclass.yaml -``` - -Test the storage class: - -```bash -# Create a test PVC -cat < Date: Mon, 13 Jul 2026 15:34:23 +1000 Subject: [PATCH 2/2] doc: fix vitepress sidebar (option 2 of 2) Order the pages in the sidebar so that they are (perfectly) congruent with the schema agreed upon between Barbara and SAP. This is the second of two options I am creating on 13 Jul 2026. This option moves the Prysm page so that it appears between the Ceph page and the Chorus page in the "Cloud Storage" hierarchy. The second option is absolutely congruent with the schema agreed upon between Barbara and SAP. In the first option (not this one), the Prysm page remains in the "Observability" folder. This is the location in which I found the Prysm page when I rearranged the other pages. It is reasonable to group the Prysm page under the rubric of "Observability", so in this commit I am leaving it where it is. Either this commit (the second option) or the other commit (the first option) should be merged. The commit that is not merged should be rejected. Both options cannot be merged without creating a merge conflict. Signed-off-by: Zac Dover --- docs/architecture/cloud-storage/observability/perses.md | 1 + docs/architecture/cloud-storage/observability/prometheus.md | 1 + docs/architecture/cloud-storage/{observability => }/prysm.md | 1 + 3 files changed, 3 insertions(+) rename docs/architecture/cloud-storage/{observability => }/prysm.md (99%) diff --git a/docs/architecture/cloud-storage/observability/perses.md b/docs/architecture/cloud-storage/observability/perses.md index 36d1f5c..a169e22 100644 --- a/docs/architecture/cloud-storage/observability/perses.md +++ b/docs/architecture/cloud-storage/observability/perses.md @@ -1,5 +1,6 @@ --- title: Perses +order: 20 --- # Perses diff --git a/docs/architecture/cloud-storage/observability/prometheus.md b/docs/architecture/cloud-storage/observability/prometheus.md index c85bb54..1dc311f 100644 --- a/docs/architecture/cloud-storage/observability/prometheus.md +++ b/docs/architecture/cloud-storage/observability/prometheus.md @@ -1,5 +1,6 @@ --- title: Prometheus +order: 10 --- # Prometheus diff --git a/docs/architecture/cloud-storage/observability/prysm.md b/docs/architecture/cloud-storage/prysm.md similarity index 99% rename from docs/architecture/cloud-storage/observability/prysm.md rename to docs/architecture/cloud-storage/prysm.md index 47b02b6..83384a9 100644 --- a/docs/architecture/cloud-storage/observability/prysm.md +++ b/docs/architecture/cloud-storage/prysm.md @@ -1,5 +1,6 @@ --- title: Prysm +order: 35 --- # Prysm