From c228ea977fdfa70c689c85c0e3700931b7226202 Mon Sep 17 00:00:00 2001 From: Davinder Pal Date: Tue, 20 Feb 2024 21:10:58 +0200 Subject: [PATCH 1/4] #6 fix log4j issue * replace log4j with logback.xml --- inventory/development/group_vars/all.yml | 3 +- readme.md | 21 +++- roles/cloudwatch/tasks/main.yml | 2 +- roles/configure/tasks/main.yml | 6 +- roles/configure/templates/log4j.properties | 58 ----------- roles/configure/templates/logback.xml | 115 +++++++++++++++++++++ 6 files changed, 138 insertions(+), 67 deletions(-) delete mode 100644 roles/configure/templates/log4j.properties create mode 100644 roles/configure/templates/logback.xml diff --git a/inventory/development/group_vars/all.yml b/inventory/development/group_vars/all.yml index a6979fa..a919b71 100755 --- a/inventory/development/group_vars/all.yml +++ b/inventory/development/group_vars/all.yml @@ -15,6 +15,7 @@ zookeeperUserId: 6000 zookeeperInstallDir: "/zookeeper" zookeeperDataDir: "{{ zookeeperInstallDir }}/zookeeper-data" zookeeperLogDir: "{{ zookeeperInstallDir }}/zookeeper-logs" +zookeeperLogLevel: "DEBUG" # DEBUG/INFO/WARN/ERROR/FATAL # zookeeper user generated MyID + FQDN for zoo.cfg are useful incase of MTLS zookeeperUserGeneratedMyId: true @@ -71,7 +72,7 @@ zookeeperVersion: 3.8.0 zookeeperOldVersion: 3.7.1 # only used in removing old versions # use local tar only -zookeeperTarLocation: "/home/davinderpal/projects/zookeeper-cluster-ansible/apache-zookeeper-{{ zookeeperVersion }}-bin.tar.gz" +zookeeperTarLocation: "/home/pox/zookeeper-cluster-ansible/apache-zookeeper-{{ zookeeperVersion }}-bin.tar.gz" # splunk/Cloudwatch monitoring zookeeperEnvironment: "development" diff --git a/readme.md b/readme.md index 787d302..bbaec06 100644 --- a/readme.md +++ b/readme.md @@ -15,12 +15,24 @@ It is group of playbooks to manage apache zookeeper. ``` ## **Development Environment Setup** + +* **STEP-0** +[Vagrant-Readme.md](./docs/vagrant-notes.md) + * **STEP-1** ``` -vagrant plugin install vagrant-hosts vagrant up ``` +Generate MTLS Certs/JKS Files +```bash +mkdir files/certs/ + +cd files/certs/ + +../vagrant-generate-tls-certs.sh +``` + * **STEP-2** ``` ansible-playbook -i inventory/development/cluster.ini clusterSetup.yml @@ -70,7 +82,7 @@ It will enable following things on all nodes. ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=zoo.cfg ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=java.env ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=jaas.conf -ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=log4j.properties +ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=logback.xml ``` ### **To upgrade java version of cluster** @@ -101,6 +113,7 @@ ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml ### **Tested Zookeeper Versions** * `3.7.1` * `3.8.0` +* `3.9.1` ### **Tested OS** * CentOS 7 @@ -110,6 +123,6 @@ ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml ### **Tested Ansible Version** ``` -ansible==6.1.0 -ansible-core==2.13.2 +ansible==9.2.0 +ansible-core==2.16.3 ``` diff --git a/roles/cloudwatch/tasks/main.yml b/roles/cloudwatch/tasks/main.yml index bd7c552..f657462 100644 --- a/roles/cloudwatch/tasks/main.yml +++ b/roles/cloudwatch/tasks/main.yml @@ -13,7 +13,7 @@ [cloudwatch-zookeeper-logs] time_zone = LOCAL datetime_format = %b %d %H:%M:%S - file = {{ zookeeperInstallDir }}/zookeeper-logs/*.out + file = {{ zookeeperInstallDir }}/zookeeper-logs/* buffer_duration = 5000 log_stream_name = {instance_id} initial_position = start_of_file diff --git a/roles/configure/tasks/main.yml b/roles/configure/tasks/main.yml index a0c6174..c521a08 100644 --- a/roles/configure/tasks/main.yml +++ b/roles/configure/tasks/main.yml @@ -10,9 +10,9 @@ name: configure tasks_from: dynamicConfigs.yml vars: - - zookeeperConfigFile: "{{ item }}" + zookeeperConfigFile: "{{ item }}" loop: - zoo.cfg - - log4j.properties - java.env - - jaas.conf \ No newline at end of file + - jaas.conf + - logback.xml diff --git a/roles/configure/templates/log4j.properties b/roles/configure/templates/log4j.properties deleted file mode 100644 index 73e29eb..0000000 --- a/roles/configure/templates/log4j.properties +++ /dev/null @@ -1,58 +0,0 @@ -# Define some default values that can be overridden by system properties -zookeeper.root.logger=INFO, CONSOLE -zookeeper.console.threshold=INFO -zookeeper.log.dir={{ zookeeperLogDir }} -zookeeper.log.file=zookeeper.log -zookeeper.log.threshold=DEBUG -zookeeper.tracelog.dir={{ zookeeperLogDir }} -zookeeper.tracelog.file=zookeeper_trace.log - -# -# ZooKeeper Logging Configuration -# - -# Format is " (, )+ - -# DEFAULT: console appender only -log4j.rootLogger=${zookeeper.root.logger} - -# Example with rolling log file -#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE - -# Example with rolling log file and tracing -#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE - -# -# Log INFO level and above messages to the console -# -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold} -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - -# -# Add ROLLINGFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender -log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold} -log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file} - -# Max log file size of 10MB -log4j.appender.ROLLINGFILE.MaxFileSize=10MB -# uncomment the next line to limit number of backup files -#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 - -log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout -log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n - - -# -# Add TRACEFILE to rootLogger to get log file output -# Log DEBUG level and above messages to a log file -log4j.appender.TRACEFILE=org.apache.log4j.FileAppender -log4j.appender.TRACEFILE.Threshold=TRACE -log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file} - -log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout -### Notice we are including log4j's NDC here (%x) -log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/roles/configure/templates/logback.xml b/roles/configure/templates/logback.xml new file mode 100644 index 0000000..b29133a --- /dev/null +++ b/roles/configure/templates/logback.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.console.threshold} + + + + + + ${zookeeper.log.dir}/${zookeeper.log.file} + + %d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n + + + ${zookeeper.log.threshold} + + + ${zookeeper.log.maxbackupindex} + ${zookeeper.log.dir}/${zookeeper.log.file}.%i + + + ${zookeeper.log.maxfilesize} + + + + + + + + + + + + + + \ No newline at end of file From b2d37cfd36b60bddbe506d6cdcd10b7984d1fcca Mon Sep 17 00:00:00 2001 From: Davinder Pal Date: Tue, 20 Feb 2024 21:16:04 +0200 Subject: [PATCH 2/4] * fix issue with zkCli.sh not working with sasl * updated readme for sasl setup * logging updates to dependent systesms likes splunk --- Splunk-Config.md | 2 +- Vagrantfile | 1 + clusterMigrateToMtls.yml | 12 ++++++------ clusterMigrateToSasLAuth.yml | 10 +++++----- docs/migrate-to-mtls.md | 1 + docs/vagrant-notes.md | 8 ++++++++ files/vagrant-generate-tls-certs.sh | 0 roles/configure/templates/jaas.conf | 11 +++++++++++ 8 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 docs/vagrant-notes.md mode change 100644 => 100755 files/vagrant-generate-tls-certs.sh diff --git a/Splunk-Config.md b/Splunk-Config.md index e3aad29..4d4815e 100755 --- a/Splunk-Config.md +++ b/Splunk-Config.md @@ -5,7 +5,7 @@ [default] host = $HOSTNAME -[monitor:///zookeeper/zookeeper-logs/*.out] +[monitor:///zookeeper/zookeeper-logs/*] disabled = false index = kafka sourcetype = zookeeper diff --git a/Vagrantfile b/Vagrantfile index 56177d9..b0dc03e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -22,4 +22,5 @@ Vagrant.configure("2") do |config| # SSH config to use your local ssh key for auth instead of username/password config.ssh.insert_key = false config.vm.provision "file", source: "~/.ssh/id_rsa.pub", destination: "~/.ssh/authorized_keys" + config.vm.synced_folder '.', '/vagrant', disabled: true end diff --git a/clusterMigrateToMtls.yml b/clusterMigrateToMtls.yml index f6a3be4..c3d4180 100644 --- a/clusterMigrateToMtls.yml +++ b/clusterMigrateToMtls.yml @@ -63,9 +63,9 @@ name: configure tasks_from: dynamicConfigs vars: - - zookeeperConfigFile: zoo.cfg - - zookeeperSslQuorum: true - - zookeeperPortUnification: "true" # force true + zookeeperConfigFile: zoo.cfg + zookeeperSslQuorum: true + zookeeperPortUnification: "true" # force true - name: MigrateToMtls | restarting zookeeper ansible.builtin.import_role: @@ -100,9 +100,9 @@ name: configure tasks_from: dynamicConfigs vars: - - zookeeperConfigFile: zoo.cfg - - zookeeperSslQuorum: true - - zookeeperPortUnification: "false" # force false + zookeeperConfigFile: zoo.cfg + zookeeperSslQuorum: true + zookeeperPortUnification: "false" # force false - name: MigrateToMtls | restarting zookeeper ansible.builtin.import_role: diff --git a/clusterMigrateToSasLAuth.yml b/clusterMigrateToSasLAuth.yml index 97a45c3..20831ea 100644 --- a/clusterMigrateToSasLAuth.yml +++ b/clusterMigrateToSasLAuth.yml @@ -14,15 +14,15 @@ name: configure tasks_from: dynamicConfigs vars: - - zookeeperConfigFile: jaas.conf + zookeeperConfigFile: jaas.conf - name: MigrateToSasL | regenerate java.env to enable jaas.conf ansible.builtin.include_role: name: configure tasks_from: dynamicConfigs vars: - - zookeeperConfigFile: java.env - - zookeeperQuorumAuthEnableSasl: true + zookeeperConfigFile: java.env + zookeeperQuorumAuthEnableSasl: true - name: MigrateToSasL | enableSasl in zoo.cfg ansible.builtin.lineinfile: @@ -83,8 +83,8 @@ name: configure tasks_from: dynamicConfigs vars: - - zookeeperConfigFile: zoo.cfg - - zookeeperQuorumAuthEnableSasl: true + zookeeperConfigFile: zoo.cfg + zookeeperQuorumAuthEnableSasl: true - name: MigrateToSasL | restarting zookeeper ansible.builtin.import_role: diff --git a/docs/migrate-to-mtls.md b/docs/migrate-to-mtls.md index 8d91934..135114b 100644 --- a/docs/migrate-to-mtls.md +++ b/docs/migrate-to-mtls.md @@ -4,6 +4,7 @@ Read documentation here: https://zookeeper.apache.org/doc/r3.8.0/zookeeperAdmin. ### Step 0 Generate MTLS Certs, if you are testing with vagrant then you can use below-mentioned script else read above-mentioned documenations. +The following script generates certs in the directory from where you are running the script. [vagrant-generate-tls-certs.sh](../files/vagrant-generate-tls-certs.sh) diff --git a/docs/vagrant-notes.md b/docs/vagrant-notes.md new file mode 100644 index 0000000..6e81836 --- /dev/null +++ b/docs/vagrant-notes.md @@ -0,0 +1,8 @@ +## Running on Windows + +### Requires following plugins +```bash +vagrant plugin install vagrant-hosts +vagrant plugin install virtualbox_WSL2 +vagrant plugin install vagrant-vbguest # optional +``` \ No newline at end of file diff --git a/files/vagrant-generate-tls-certs.sh b/files/vagrant-generate-tls-certs.sh old mode 100644 new mode 100755 diff --git a/roles/configure/templates/jaas.conf b/roles/configure/templates/jaas.conf index 7251023..20347fa 100644 --- a/roles/configure/templates/jaas.conf +++ b/roles/configure/templates/jaas.conf @@ -8,3 +8,14 @@ QuorumLearner { username="{{ zookeeperQuorumUsername }}" password="{{ zookeeperQuorumPassword }}"; }; + +Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_{{ zookeeperQuorumUsername }}="{{ zookeeperQuorumPassword }}"; +}; + +Client { + org.apache.zookeeper.server.auth.DigestLoginModule required + username="{{ zookeeperQuorumUsername }}" + password="{{ zookeeperQuorumPassword }}"; +}; From 7f66d2b22d244b5e91cebddf967891777f20ac59 Mon Sep 17 00:00:00 2001 From: Davinder Pal Date: Tue, 20 Feb 2024 21:35:50 +0200 Subject: [PATCH 3/4] * rename systemUpgrade to OSUpgrade * fix issue in clusterUpgrade with Sasl --- clusterSystemUpgrade.yml => clusterOSUpgrade.yml | 0 clusterUpgrade.yml | 4 ++++ inventory/development/group_vars/all.yml | 4 ++-- readme.md | 6 +++++- roles/configure/templates/logback.xml | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) rename clusterSystemUpgrade.yml => clusterOSUpgrade.yml (100%) diff --git a/clusterSystemUpgrade.yml b/clusterOSUpgrade.yml similarity index 100% rename from clusterSystemUpgrade.yml rename to clusterOSUpgrade.yml diff --git a/clusterUpgrade.yml b/clusterUpgrade.yml index 7074fda..0cd48ac 100644 --- a/clusterUpgrade.yml +++ b/clusterUpgrade.yml @@ -16,6 +16,10 @@ name: install tasks_from: download + - name: copy mtls/sasl files + ansible.builtin.import_role: + name: copyFiles + - name: configuring latest Zookeeper version ansible.builtin.import_role: name: configure diff --git a/inventory/development/group_vars/all.yml b/inventory/development/group_vars/all.yml index a919b71..4f9a395 100755 --- a/inventory/development/group_vars/all.yml +++ b/inventory/development/group_vars/all.yml @@ -68,8 +68,8 @@ zookeeperPrometheusExporterEnabled: true zookeeperPrometheusExporterHttpPort: 7000 # zookeeper versions -zookeeperVersion: 3.8.0 -zookeeperOldVersion: 3.7.1 # only used in removing old versions +zookeeperVersion: 3.9.1 +zookeeperOldVersion: 3.8.0 # only used in removing old versions # use local tar only zookeeperTarLocation: "/home/pox/zookeeper-cluster-ansible/apache-zookeeper-{{ zookeeperVersion }}-bin.tar.gz" diff --git a/readme.md b/readme.md index bbaec06..dfffc5c 100644 --- a/readme.md +++ b/readme.md @@ -84,6 +84,10 @@ ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=jaas.conf ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=logback.xml ``` +### **To upgrade zookeeper version of cluster** +* Update Required vars in ```inventory//group_vars/all.yml``` . + +```ansible-playbook -i inventory//cluster.ini clusterUpgrade.yml``` ### **To upgrade java version of cluster** * Update Required vars in ```inventory//group_vars/all.yml``` . @@ -93,7 +97,7 @@ ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml ### **To upgrade OS version of cluster** * Update Required vars in ```inventory//group_vars/all.yml``` . -```ansible-playbook -i inventory//cluster.ini clusterSystemUpgrade.yml``` +```ansible-playbook -i inventory//cluster.ini clusterOSUpgrade.yml``` ### **To remove old version files of zookeeper from cluster** * Update Required vars in ```inventory//group_vars/all.yml``` . diff --git a/roles/configure/templates/logback.xml b/roles/configure/templates/logback.xml index b29133a..bd7c507 100644 --- a/roles/configure/templates/logback.xml +++ b/roles/configure/templates/logback.xml @@ -23,7 +23,7 @@ - + From dba021e7fb0019deec9fe95f7b4e0a3cbec8cf50 Mon Sep 17 00:00:00 2001 From: Davinder Pal Date: Tue, 20 Feb 2024 21:43:01 +0200 Subject: [PATCH 4/4] markdown style issues --- Splunk-Config.md | 7 ++++--- readme.md | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Splunk-Config.md b/Splunk-Config.md index 4d4815e..4e7bd3d 100755 --- a/Splunk-Config.md +++ b/Splunk-Config.md @@ -1,7 +1,8 @@ # Splunk Logging Configuration -**Example** -``` +## Example + +```conf [default] host = $HOSTNAME @@ -10,4 +11,4 @@ disabled = false index = kafka sourcetype = zookeeper crcSalt = -``` \ No newline at end of file +``` diff --git a/readme.md b/readme.md index dfffc5c..8a0e00a 100644 --- a/readme.md +++ b/readme.md @@ -3,6 +3,7 @@ It is group of playbooks to manage apache zookeeper. ## **Requirements** + * Download Apache Zookeeper Tar Manually ( Mandatory ) * vagrant ( Optional ) * Any OS with SystemD ( Mandatory ) @@ -10,6 +11,7 @@ It is group of playbooks to manage apache zookeeper. * `netaddr` python package on ansible controller node. ## **Notes*** + ``` 1. All tasks like jvm/logging/downgrade/removeOldVersion will be done in serial order. ``` @@ -25,6 +27,7 @@ vagrant up ``` Generate MTLS Certs/JKS Files + ```bash mkdir files/certs/ @@ -34,7 +37,8 @@ cd files/certs/ ``` * **STEP-2** -``` + +```bash ansible-playbook -i inventory/development/cluster.ini clusterSetup.yml ``` @@ -46,6 +50,7 @@ ansible-playbook -i inventory/development/cluster.ini clusterSetup.yml * `terraform/oci` ### **AWS Cloud PreSetup for cluster** + It will enable following things on all nodes. 1. `/zookeeper` mount point from ebs created by terraform. @@ -58,12 +63,14 @@ It will enable following things on all nodes. ```ansible-playbook -i inventory//cluster.ini clusterAwsPreSetup.yml``` ### **To start new cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . * Update Required vars in ```inventory//cluster.ini``` . ```ansible-playbook -i inventory//cluster.ini clusterSetup.yml``` ### **Monitoring Setup** + * **To add custom metric exporter to cluster** ```ansible-playbook -i inventory//cluster.ini clusterCustomMetricExporter.yml``` @@ -77,55 +84,69 @@ It will enable following things on all nodes. ```ansible-playbook -i inventory//cluster.ini clusterRollingRestart.yml``` ### **To update jvm/logging/zoo.cg/jaas.conf settings of cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . + ```bash ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=zoo.cfg ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=java.env ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=jaas.conf ansible-playbook -i inventory//cluster.ini clusterConfigsUpdate.yml -e zookeeperConfigFile=logback.xml ``` + ### **To upgrade zookeeper version of cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . ```ansible-playbook -i inventory//cluster.ini clusterUpgrade.yml``` ### **To upgrade java version of cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . ```ansible-playbook -i inventory//cluster.ini clusterJava.yml``` ### **To upgrade OS version of cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . ```ansible-playbook -i inventory//cluster.ini clusterOSUpgrade.yml``` ### **To remove old version files of zookeeper from cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . ```ansible-playbook -i inventory//cluster.ini clusterRemoveOldVersion.yml``` ### **To remove zookeeper cluster** + * Update Required vars in ```inventory//group_vars/all.yml``` . ```ansible-playbook -i inventory//cluster.ini clusterRemoveNodes.yml``` ## **Migration Playbooks** + ### [Migrate Zookeeper to FQDN based Configurations](./docs/migrate-to-fqdn-based-configs.md) + ### [Migrate Zookeeper to SASL Cluster](./docs/migrate-to-sasl.md) + ### [Migrate Zookeeper to MTLS Quorum Cluster](./docs/migrate-to-mtls.md) ### **Tested Zookeeper Versions** + * `3.7.1` * `3.8.0` * `3.9.1` ### **Tested OS** + * CentOS 7 * RedHat 7 * Amzaon Linux 2 * Ubuntu 18 ### **Tested Ansible Version** + ``` ansible==9.2.0 ansible-core==2.16.3