[multiple] Discover and configure Glance with Ceph RGW secrets

This patch enhances the cifmw_ceph_client/cifmw_cephadm roles to:
- Automatically discover Ceph RGW (RADOS Gateway) endpoint and credentials
- Create Glance secrets using the discovered RGW settings
- Configure glance with Ceph RGW secrets
- Creating a dedicated 'glance' S3 user in RGW (cifmw_cephadm role)

This integration configures Glance with Ceph RGW access credentials,
enabling it to use RGW as an Glance backend when object storage is enabled

Changes:
- Add RGW discovery playbook
- Add logic to create Glance secrets with RGW config

Signed-off-by: Maxim Sava <msava@redhat.com>

Author: Maxim Sava

hooks/playbooks/ceph.yml

@@ -473,6 +473,12 @@
         name: cifmw_cephadm
         tasks_from: rbd_mirror
 
+    - name: Create RGW S3 user and fetch S3 user info
+      when: cifmw_cephadm_rgw_s3_glance
+      ansible.builtin.import_role:
+        name: cifmw_cephadm
+        tasks_from: glance_s3_info
+
     - name: Create Cephx Keys for OpenStack
       ansible.builtin.import_role:
         name: cifmw_cephadm

roles/cifmw_ceph_client/tasks/glance-rgw-discovery.yaml

@@ -0,0 +1,60 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Set fact for glance S3 backend secrets
+  ansible.builtin.set_fact:
+    ceph_s3_access_key: "{{ cifmw_ceph_client_s3_glance_user['keys'][0]['access_key'] }}"
+    ceph_s3_secret_key: "{{ cifmw_ceph_client_s3_glance_user['keys'][0]['secret_key'] }}"
+    ceph_s3_bucket: "glance-s3-bucket"
+    ceph_s3_endpoint: "{{ cifmw_ceph_client_s3_internal_swift_url }}"
+  when:
+    - cifmw_ceph_client_s3_glance_user is defined
+    - cifmw_ceph_client_s3_internal_swift_url is defined
+  no_log: true
+
+- name: Generate Glance S3 backend secrets using Ceph RGW credentials
+  ansible.builtin.template:
+    src: templates/k8s_ceph_rgw_glance_secret.j2
+    dest: k8s_ceph_rgw_glance_secret.yaml
+    mode: "0600"
+    force: true
+  when:
+    - ceph_s3_access_key is defined
+    - ceph_s3_secret_key is defined
+  no_log: true
+  register: reg_k8s_ceph_rgw_glance_secret
+
+- name: Create glance S3 backend secrets
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    namespace: openstack
+    state: present
+    apply: true
+    force: true
+    src: k8s_ceph_rgw_glance_secret.yaml
+  register: k8s_glance_s3_secret
+  delegate_to: localhost
+  no_log: true
+  when: reg_k8s_ceph_rgw_glance_secret is not skipped
+
+- name: Delete the rendered secret file
+  ansible.builtin.file:
+    path: k8s_ceph_rgw_glance_secret.yaml
+    state: absent
+  delegate_to: localhost
+  when: k8s_glance_s3_secret | default(true)

roles/cifmw_ceph_client/tasks/main.yml

@@ -78,6 +78,10 @@
     mode: "0600"
     force: true
 
+- name: Import glance Ceph RGW discovery playbook
+  when: cifmw_ceph_client_s3_glance_user
+  ansible.builtin.include_tasks: glance-rgw-discovery.yaml
+
 - name: Create edpm-values-post-ceph ConfigMap if sample path provided
   ansible.builtin.include_tasks: edpm_values_post_ceph.yml
   when:

roles/cifmw_ceph_client/templates/k8s_ceph_rgw_glance_secret.j2

@@ -0,0 +1,13 @@
+# Glance secrets used to configure glance with ceph S3 backend
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3glance
+stringData:
+  s3glance.conf: |
+    [default_backend]
+    s3_store_host = {{ ceph_s3_endpoint }}
+    s3_store_access_key = {{ ceph_s3_access_key }}
+    s3_store_secret_key = {{ ceph_s3_secret_key }}
+    s3_store_bucket = {{ ceph_s3_bucket }}
+    s3_store_cacert = "/etc/pki/tls/certs/ca-bundle.crt"

roles/cifmw_cephadm/README.md

@@ -108,6 +108,9 @@ that they do not need to be changed for a typical EDPM deployment.
   Squid). Set it to false if the target Ceph release is equal to or greater
   than Tentacle.
 
+* `cifmw_cephadm_rgw_s3_glance`: (Bool) If this is value is `true`, then
+  cephadm will create glance secrets using the discovered RGW settings
+
 Use the `cifmw_cephadm_pools` list of dictionaries to define pools for
 Nova (vms), Cinder (volumes), Cinder-backups (backups), and Glance (images).
 ```

roles/cifmw_cephadm/defaults/main.yml

@@ -156,3 +156,4 @@ cifmw_cephadm_rgw_ingress_service_id: "rgw.default"
 # set ssl_backward compatibily to False if ceph version is equal or greater
 # than Tentacle
 cifmw_rgw_ssl_backward_compatibility: true
+cifmw_cephadm_rgw_s3_glance: false

roles/cifmw_cephadm/tasks/glance_s3_info.yaml

@@ -0,0 +1,124 @@
+---
+ # Copyright 2025 Red Hat, Inc.
+ # All Rights Reserved.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may
+ # not use this file except in compliance with the License. You may obtain
+ # a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ # License for the specific language governing permissions and limitations
+ # under the License.
+
+- name: Create glance S3 RGW user and fetch object-store endpoint
+  when: cifmw_edpm_deploy_hci | default('false') | bool
+  block:
+    - name: Refresh ceph_cli
+      ansible.builtin.include_tasks: ceph_cli.yml
+      vars:
+        ceph_command: "radosgw-admin"
+
+    - name: Wait for RGW daemons to be running
+      ansible.builtin.command:
+        cmd: "{{ cifmw_cephadm_ceph_cli }} orch ps --daemon-type rgw --format json"
+      register: rgw_daemon_status
+      become: true
+      changed_when: false
+      failed_when: false
+      until: >
+        rgw_daemon_status.rc == 0 and
+        (rgw_daemon_status.stdout | default('[]') | from_json | default([], true)) |
+        selectattr('status_desc', 'equalto', 'running') |
+        list | length > 0
+      retries: 30
+      delay: 10
+
+    - name: Check if Ceph S3 glance user exists
+      ansible.builtin.command:
+        cmd: >-
+          {{ cifmw_cephadm_ceph_cli }} user info --uid glance
+      no_log: true
+      become: true
+      ignore_errors: true
+      changed_when: false
+      register: cifmw_ceph_s3_glance_user_check
+      when: cifmw_cephadm_ceph_cli is defined
+
+    - name: Create Ceph S3 glance user
+      ansible.builtin.command:
+        cmd: >-
+          {{ cifmw_cephadm_ceph_cli }} user create
+          --uid="glance"
+          --display-name="Glance S3 User"
+      become: true
+      when:
+        - cifmw_ceph_s3_glance_user_check is not skipped
+        - cifmw_ceph_s3_glance_user_check.rc != 0
+      register: glance_rgw_user_create
+      retries: 3
+      delay: 5
+      until: glance_rgw_user_create.rc == 0
+
+    - name: Get RGW glance user info
+      ansible.builtin.command:
+        cmd: >-
+          {{ cifmw_cephadm_ceph_cli }} user info --uid="glance"
+      become: true
+      no_log: true
+      changed_when: false
+      register: ceph_rgw_glance_user_info
+
+    - name: Show RGW daemon status on failure
+      when: ceph_rgw_glance_user_info.rc != 0
+      block:
+        - name: Get RGW daemon status for debugging
+          ansible.builtin.command:
+            cmd: "{{ cifmw_cephadm_ceph_cli }} orch ps --daemon-type rgw"
+          become: true
+          register: rgw_debug_status
+          changed_when: false
+
+        - name: Display RGW status
+          ansible.builtin.debug:
+            msg: "RGW daemon status: {{ rgw_debug_status.stdout }}"
+
+        - name: Fail with context
+          ansible.builtin.fail:
+            msg: |
+              Failed to fetch glance user info after creation.
+              User creation result: {{ glance_rgw_user_create | default('not attempted') }}
+              RGW daemon status shown above.
+
+    - name: Set facts RGW glance user
+      ansible.builtin.set_fact:
+        cifmw_ceph_s3_glance_user: "{{ ceph_rgw_glance_user_info.stdout | from_json }}"
+      when:
+        - ceph_rgw_glance_user_info is defined
+        - ceph_rgw_glance_user_info.rc == 0
+      no_log: true
+
+    - name: Get OpenStack internal object-store endpoints
+      cifmw.general.ci_script:
+        extra_args:
+          KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        output_dir: "{{ cifmw_cephadm_basedir }}/artifacts"
+        script: |-
+          oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint list --service object-store --interface internal -f value -c URL
+      delegate_to: localhost
+      register: object_store_endpoint_raw
+      when:
+        - cifmw_openshift_kubeconfig is defined
+        - cifmw_cephadm_basedir is defined
+        - cifmw_cephadm_rgw_s3_glance
+
+    - name: Extract base URL without path
+      ansible.builtin.set_fact:
+        rgw_internal_url: >-
+          {{
+            object_store_endpoint_raw.stdout
+            | regex_replace('(/swift.*)$', '')
+          }}

roles/cifmw_cephadm/templates/ceph_client.yaml.j2

@@ -1,4 +1,10 @@
 ---
+{% if cifmw_cephadm_rgw_s3_glance is defined %}
+cifmw_ceph_client_s3_glance_user: {{ cifmw_cephadm_rgw_s3_glance }}
+{% endif %}
+{% if rgw_internal_url is defined %}
+cifmw_ceph_client_s3_internal_swift_url: {{ rgw_internal_url }}
+{% endif %}
 cifmw_ceph_client_fsid: {{ cifmw_cephadm_fsid }}
 cifmw_ceph_client_cluster: {{ cifmw_cephadm_cluster }}
 cifmw_ceph_client_external_cluster_mon_ips: "{{ cifmw_ceph_client_external_cluster_mon_ips }}"