Updates and Bugfixes

Splunk-Config.md

@@ -1,13 +1,14 @@
 # Splunk Logging Configuration
 
-**Example**
-```
+## Example
+
+```conf
 [default]
 host = $HOSTNAME
 
-[monitor:///zookeeper/zookeeper-logs/*.out]
+[monitor:///zookeeper/zookeeper-logs/*]
 disabled = false
 index = kafka
 sourcetype = zookeeper
 crcSalt = <SOURCE>
-```
+```

Vagrantfile

@@ -22,4 +22,5 @@ Vagrant.configure("2") do |config|
 # SSH config to use your local ssh key for auth instead of username/password
   config.ssh.insert_key = false
   config.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "~/.ssh/authorized_keys"
+  config.vm.synced_folder '.', '/vagrant', disabled: true
 end

clusterMigrateToMtls.yml

@@ -63,9 +63,9 @@
         name: configure
         tasks_from: dynamicConfigs
       vars:
-        - zookeeperConfigFile: zoo.cfg
-        - zookeeperSslQuorum: true
-        - zookeeperPortUnification: "true" # force true
+        zookeeperConfigFile: zoo.cfg
+        zookeeperSslQuorum: true
+        zookeeperPortUnification: "true" # force true
 
     - name: MigrateToMtls | restarting zookeeper
       ansible.builtin.import_role:
@@ -100,9 +100,9 @@
         name: configure
         tasks_from: dynamicConfigs
       vars:
-        - zookeeperConfigFile: zoo.cfg
-        - zookeeperSslQuorum: true
-        - zookeeperPortUnification: "false" # force false
+        zookeeperConfigFile: zoo.cfg
+        zookeeperSslQuorum: true
+        zookeeperPortUnification: "false" # force false
 
     - name: MigrateToMtls | restarting zookeeper
       ansible.builtin.import_role:

clusterMigrateToSasLAuth.yml

@@ -14,15 +14,15 @@
         name: configure
         tasks_from: dynamicConfigs
       vars:
-        - zookeeperConfigFile: jaas.conf
+        zookeeperConfigFile: jaas.conf
 
     - name: MigrateToSasL | regenerate java.env to enable jaas.conf
       ansible.builtin.include_role:
         name: configure
         tasks_from: dynamicConfigs
       vars:
-        - zookeeperConfigFile: java.env
-        - zookeeperQuorumAuthEnableSasl: true
+        zookeeperConfigFile: java.env
+        zookeeperQuorumAuthEnableSasl: true
 
     - name: MigrateToSasL | enableSasl in zoo.cfg
       ansible.builtin.lineinfile:
@@ -83,8 +83,8 @@
         name: configure
         tasks_from: dynamicConfigs
       vars:
-        - zookeeperConfigFile: zoo.cfg
-        - zookeeperQuorumAuthEnableSasl: true
+        zookeeperConfigFile: zoo.cfg
+        zookeeperQuorumAuthEnableSasl: true
 
     - name: MigrateToSasL | restarting zookeeper
       ansible.builtin.import_role:

clusterUpgrade.yml

@@ -16,6 +16,10 @@
         name: install
         tasks_from: download
 
+    - name: copy mtls/sasl files
+      ansible.builtin.import_role:
+        name: copyFiles
+
     - name: configuring latest Zookeeper version
       ansible.builtin.import_role:
         name: configure

docs/migrate-to-mtls.md

@@ -4,6 +4,7 @@ Read documentation here: https://zookeeper.apache.org/doc/r3.8.0/zookeeperAdmin.
 
 ### Step 0
 Generate MTLS Certs, if you are testing with vagrant then you can use below-mentioned script else read above-mentioned documenations.
+The following script generates certs in the directory from where you are running the script.
 
 [vagrant-generate-tls-certs.sh](../files/vagrant-generate-tls-certs.sh)
 

docs/vagrant-notes.md

@@ -0,0 +1,8 @@
+## Running on Windows
+
+### Requires following plugins
+```bash
+vagrant plugin install vagrant-hosts
+vagrant plugin install virtualbox_WSL2
+vagrant plugin install vagrant-vbguest # optional
+```

inventory/development/group_vars/all.yml

@@ -15,6 +15,7 @@ zookeeperUserId: 6000
 zookeeperInstallDir: "/zookeeper"
 zookeeperDataDir: "{{ zookeeperInstallDir }}/zookeeper-data"
 zookeeperLogDir: "{{ zookeeperInstallDir }}/zookeeper-logs"
+zookeeperLogLevel: "DEBUG"    # DEBUG/INFO/WARN/ERROR/FATAL
 
 # zookeeper user generated MyID + FQDN for zoo.cfg are useful incase of MTLS
 zookeeperUserGeneratedMyId: true
@@ -67,11 +68,11 @@ zookeeperPrometheusExporterEnabled: true
 zookeeperPrometheusExporterHttpPort: 7000
 
 # zookeeper versions
-zookeeperVersion: 3.8.0
-zookeeperOldVersion: 3.7.1 # only used in removing old versions
+zookeeperVersion: 3.9.1
+zookeeperOldVersion: 3.8.0 # only used in removing old versions
 
 # use local tar only
-zookeeperTarLocation: "/home/davinderpal/projects/zookeeper-cluster-ansible/apache-zookeeper-{{ zookeeperVersion }}-bin.tar.gz"
+zookeeperTarLocation: "/home/pox/zookeeper-cluster-ansible/apache-zookeeper-{{ zookeeperVersion }}-bin.tar.gz"
 
 # splunk/Cloudwatch monitoring
 zookeeperEnvironment: "development"

readme.md

@@ -3,26 +3,42 @@
 It is group of playbooks to manage apache zookeeper.
 
 ## **Requirements**
+
 * Download Apache Zookeeper Tar Manually ( Mandatory )
 * vagrant ( Optional )
 * Any OS with SystemD ( Mandatory )
 * Ansible ( Mandatory )
 * `netaddr` python package on ansible controller node.
 
 ## **Notes***
+
 ```
 1. All tasks like jvm/logging/downgrade/removeOldVersion will be done in serial order.
 ```
 
 ## **Development Environment Setup**
+
+* **STEP-0**
+[Vagrant-Readme.md](./docs/vagrant-notes.md)
+
 * **STEP-1**
 ```
-vagrant plugin install vagrant-hosts
 vagrant up
 ```
 
-* **STEP-2**
+Generate MTLS Certs/JKS Files
+
+```bash
+mkdir files/certs/
+
+cd files/certs/
+
+../vagrant-generate-tls-certs.sh
 ```
+
+* **STEP-2**
+
+```bash
 ansible-playbook -i inventory/development/cluster.ini clusterSetup.yml
 ```
 
@@ -34,6 +50,7 @@ ansible-playbook -i inventory/development/cluster.ini clusterSetup.yml
 * `terraform/oci`
 
 ### **AWS Cloud PreSetup for cluster**
+
 It will enable following things on all nodes.
 
 1. `/zookeeper` mount point from ebs created by terraform.
@@ -46,12 +63,14 @@ It will enable following things on all nodes.
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterAwsPreSetup.yml```
 
 ### **To start new cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 * Update Required vars in ```inventory/<environment>/cluster.ini``` .
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterSetup.yml```
 
 ### **Monitoring Setup**
+
 * **To add custom metric exporter to cluster**
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterCustomMetricExporter.yml```
@@ -65,51 +84,70 @@ It will enable following things on all nodes.
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterRollingRestart.yml```
 
 ### **To update jvm/logging/zoo.cg/jaas.conf settings of cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
+
 ```bash
 ansible-playbook -i inventory/<environment>/cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=zoo.cfg
 ansible-playbook -i inventory/<environment>/cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=java.env
 ansible-playbook -i inventory/<environment>/cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=jaas.conf
-ansible-playbook -i inventory/<environment>/cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=log4j.properties
+ansible-playbook -i inventory/<environment>/cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=logback.xml
 ```
 
+### **To upgrade zookeeper version of cluster**
+
+* Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
+
+```ansible-playbook -i inventory/<environment>/cluster.ini clusterUpgrade.yml```
+
 ### **To upgrade java version of cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterJava.yml```
 
 ### **To upgrade OS version of cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 
-```ansible-playbook -i inventory/<environment>/cluster.ini clusterSystemUpgrade.yml```
+```ansible-playbook -i inventory/<environment>/cluster.ini clusterOSUpgrade.yml```
 
 ### **To remove old version files of zookeeper from cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterRemoveOldVersion.yml```
 
 ### **To remove zookeeper cluster**
+
 * Update Required vars in ```inventory/<environment>/group_vars/all.yml``` .
 
 ```ansible-playbook -i inventory/<environment>/cluster.ini clusterRemoveNodes.yml```
 
 ## **Migration Playbooks**
+
 ### [Migrate Zookeeper to FQDN based Configurations](./docs/migrate-to-fqdn-based-configs.md)
+
 ### [Migrate Zookeeper to SASL Cluster](./docs/migrate-to-sasl.md)
+
 ### [Migrate Zookeeper to MTLS Quorum Cluster](./docs/migrate-to-mtls.md)
 
 ### **Tested Zookeeper Versions**
+
 * `3.7.1`
 * `3.8.0`
+* `3.9.1`
 
 ### **Tested OS**
+
 * CentOS 7
 * RedHat 7
 * Amzaon Linux 2
 * Ubuntu 18
 
 ### **Tested Ansible Version**
+
 ```
-ansible==6.1.0
-ansible-core==2.13.2
+ansible==9.2.0
+ansible-core==2.16.3
 ```

roles/cloudwatch/tasks/main.yml

@@ -13,7 +13,7 @@
       [cloudwatch-zookeeper-logs]
       time_zone = LOCAL
       datetime_format = %b %d %H:%M:%S
-      file = {{ zookeeperInstallDir }}/zookeeper-logs/*.out
+      file = {{ zookeeperInstallDir }}/zookeeper-logs/*
       buffer_duration = 5000
       log_stream_name = {instance_id}
       initial_position = start_of_file

roles/configure/tasks/main.yml

@@ -10,9 +10,9 @@
       name: configure
       tasks_from: dynamicConfigs.yml
   vars:
-    - zookeeperConfigFile: "{{ item }}"
+    zookeeperConfigFile: "{{ item }}"
   loop:
     - zoo.cfg
-    - log4j.properties
     - java.env
-    - jaas.conf
+    - jaas.conf
+    - logback.xml

roles/configure/templates/jaas.conf

@@ -8,3 +8,14 @@ QuorumLearner {
        username="{{ zookeeperQuorumUsername }}"
        password="{{ zookeeperQuorumPassword }}";
 };
+
+Server {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       user_{{ zookeeperQuorumUsername }}="{{ zookeeperQuorumPassword }}";
+};
+
+Client {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       username="{{ zookeeperQuorumUsername }}"
+       password="{{ zookeeperQuorumPassword }}";
+};