ADEI cachers

2 files changed, 121 insertions, 63 deletions

diff --git a/setup/projects/adei/templates/60-adei.yml.j2 b/setup/projects/adei/templates/60-adei.yml.j2
index 88bfa89..41840e4 100644
--- a/setup/projects/adei/templates/60-adei.yml.j2
+++ b/setup/projects/adei/templates/60-adei.yml.j2
@@ -9,7 +9,7 @@ metadata:
     openshift.io/documentation-url: "http://adei.info"
     openshift.io/support-url: "http://adei.info"
 labels:
-  name: adei
+  app: "adei"
 objects:
 {% for name, cfg in adei_frontends.iteritems() %}
 {%   if (cfg.enabled | default(true))  %}
@@ -62,6 +62,8 @@ objects:
     spec:
       schedule: "{{ cfg.cron }}"
       concurrencyPolicy: "Forbid"
+      successfulJobsHistoryLimit: "2"
+      failedJobsHistoryLimit: "2"
       jobTemplate:
         spec:
             completions: "1"
@@ -98,7 +100,6 @@ objects:
 {%     endif  %}
                 name: "{{ cfg.name }}"
                 labels:
-                  app: "adei"
                   name: "{{ cfg.name }}"
                   adei-type: "{{ pod_type }}"
                   adei-name: "{{ name }}"
@@ -171,31 +172,37 @@ parameters:
   - name: setup
     value: "autogen"
     description: "ADEI setup"
+  - name: continuous_caching
+    value: "1"
+    description: "Stop cachers if no work or poll the source continously"  
+  - name: enable_logs
+    value: "1"
+    description: "Enable detailed ADEI logging (large volume)"
+  - name: enable_debug
+    value: "0"
+    description: "Include debugging information in ADEI logs (huge volume)"
   - name: adei_replicas
     value: "2"
     description: "Number of frontend replics"
-  - name: cache_replicas
-    value: "1"
-    description: "Default number of backend caching replicas"
   - name: apache_servers
     value: "150"
-  - name: adei_schedule_cache
+  - name: sched_parallel
+    value: "sources"
+    description: "Basic unit of scheduling (servers or sources)"
+  - name: cache_replicas
+    value: "2"
+    description: "Default number of backend caching replicas"
+  - name: cache_parallel
+    value: "sources"
+    description: "Type of caching parallelism by a cacher replica: (sources or groups)"
+  - name: update_schedule
     value: "*/5 * * * *"
     description: "Schedule of caching group updates"
-  - name: adei_schedule_maintain
+  - name: maintain_schedule
     value: "*/15 * * * *"
     description: "Schedule of maintain tasks"
-  - name: adei_schedule_clean
+  - name: clean_schedule
     value: "9 3 * * *"
     description: "Schedule of cleaning tasks"
-  - name: cache_parallel
-    value: "source"
-    description: "Type of caching parallelism: group, source, server"
-  - name: enable_logs
-    value: "1"
-    description: "Enable detailed ADEI logging (large volume)"
-  - name: enable_debug
-    value: "0"
-    description: "Include debugging information in ADEI logs (huge volume)"
   - name: "adei_revision"
     value: "last:1"
diff --git a/setup/projects/adei/vars/globals.yml b/setup/projects/adei/vars/globals.yml
index 60cb212..91f5933 100644
--- a/setup/projects/adei/vars/globals.yml
+++ b/setup/projects/adei/vars/globals.yml
@@ -15,17 +15,18 @@ adei_pod_env:
       value: "adei"
     - name: "ADEI_PORTS"
       value: "8080"
-    - name: "ADEI_SETUP"
-      value: "${setup}"
     - name: "ADEI_ENABLED_SETUPS"
-      value: "${setup}"
-    - name: "ADEI_PARALLEL"
-      value: "${cache_parallel}"
+      value: "${setup} ${setup}-logs"
     - name: "ADEI_REVISION"
       value: "${adei_revision}"
-
+    - name: "ADEI_PATH"
+      value: "/adei/src"
+    - name: "ADEI_URL"
+      value: "http://adei-${setup}.adei.svc.cluster.local/adei"
 
 adei_prod_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}"
     - name: "ADEI_RELEASE"
       value: "1"
     - name: "ADEI_WRITE_LOGS"
@@ -36,6 +37,8 @@ adei_prod_env:
       value: "${apache_servers}"
 
 adei_log_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}-logs"
     - name: "ADEI_RELEASE"
       value: "1"
     - name: "ADEI_WRITE_LOGS"
@@ -46,6 +49,8 @@ adei_log_env:
       value: "5"
 
 adei_debug_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}"
     - name: "ADEI_RELEASE"
       value: "0"
     - name: "ADEI_WRITE_LOGS"
@@ -56,8 +61,38 @@ adei_debug_env:
       value: "5"
 
 adei_cron_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}"
     - name: "ADEI_SCHEDULER"
-      value: "1"
+      value: "${sched_parallel}"
+
+adei_ccache_env:
+    - name: "ADEI_CONTINUOUS_CACHING"
+      value: "${continuous_caching}"
+
+adei_cache_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}"
+    - name: "ADEI_PARALLEL"
+      value: "${cache_parallel}"
+    - name: "ADEI_IDLE_SLEEP"
+      value: "5"
+
+adei_arc_cache_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}"
+    - name: "ADEI_PARALLEL"
+      value: "${cache_parallel}"
+    - name: "ADEI_IDLE_SLEEP"
+      value: "300"
+
+adei_log_cache_env:
+    - name: "ADEI_SETUP"
+      value: "${setup}-logs"
+    - name: "ADEI_PARALLEL"
+      value: "${sched_parallel}"
+    - name: "ADEI_IDLE_SLEEP"
+      value: "300"
 
 
 adei_pod_vols:
@@ -79,9 +114,6 @@ adei_pod_vols:
     - name: adei-log
       persistentVolumeClaim: 
         claimName: adei-log
-#    - name: adei-ovr
-#      emptyDir: {}
-
 
 adei_prod_mounts:
     - name: adei-src
@@ -107,42 +139,7 @@ adei_pod_mounts:
       subPath: "${setup}/apache2"
       mountPath: /var/log/apache2
 
-# Only backends...
-#    - name: adei-log
-#      subPath: "${setup}/mail"
-#      mountPath: /var/spool/mail
-
-#adei_prod_pod_mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
-#adei_dbg_pod_mounts: "{{ adei_dbg_mounts | union(adei_pod_mounts) }}"
-    
 adei_frontends:
-    update:
-      name: "adei-${setup}-update"
-      cron: "${adei_schedule_cache}"
-      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei.cron.sh" ]
-      env: "{{ adei_pod_env | union(adei_cron_env) }}"
-      vols: "{{ adei_pod_vols }}"
-      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
-      groups: [ "adei" ]
-      enabled: true
-    maintain:
-      name: "adei-${setup}-maintain"
-      cron: "${adei_schedule_cache}"
-      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_manager.cron.sh" ]
-      env: "{{ adei_pod_env | union(adei_cron_env) }}"
-      vols: "{{ adei_pod_vols }}"
-      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
-      groups: [ "adei" ]
-      enabled: false
-    clean:
-      name: "adei-${setup}-clean"
-      cron: "${adei_schedule_cache}"
-      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_clean.cron.sh" ]
-      env: "{{ adei_pod_env | union(adei_cron_env) }}"
-      vols: "{{ adei_pod_vols }}"
-      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
-      groups: [ "adei" ]
-      enabled: false
     frontend:
       name: "adei-${setup}"
       node: "adei-${setup}.{{ adei_domain }}"
@@ -165,9 +162,63 @@ adei_frontends:
     logs:
       name: "adei-${setup}-logs"
       node: "adei-${setup}-logs.{{ adei_domain }}"  
-      replicas: 1
+      replicas: "${enabled_logs}"
       env: "{{ adei_pod_env | union(adei_log_env) }}"
       vols: "{{ adei_pod_vols }}"
       mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
       groups: [ "adei" ]
       enabled: false
+    cacher:
+      name: "adei-${setup}-cacher"
+      replicas: "${cache_replicas}"
+      cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh" ]
+      env: "{{ adei_pod_env | union(adei_cache_env) | union(adei_ccache_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: true
+    archive_cacher:
+      name: "adei-${setup}-archive-cacher"
+      replicas: "1"
+      cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh -m archive" ]
+      env: "{{ adei_pod_env | union(adei_arc_cache_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: false
+    log_cacher:
+      name: "adei-${setup}-log-cacher"
+      replicas: "${enabled_logs}"
+      cmd: [ "/bin/bash", "/adei/src/scripts/system/cacher.sh" ]
+      env: "{{ adei_pod_env | union(adei_log_cache_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: false
+    update:
+      name: "adei-${setup}-update"
+      cron: "${update_schedule}"
+      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei.cron.sh" ]
+      env: "{{ adei_pod_env | union(adei_cron_env) | union(adei_ccache_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: true
+    maintain:
+      name: "adei-${setup}-maintain"
+      cron: "${maintain_schedule}"
+      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_manager.cron.sh" ]
+      env: "{{ adei_pod_env | union(adei_cron_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: false
+    clean:
+      name: "adei-${setup}-clean"
+      cron: "${clean_schedule}"
+      cmd: [ "/bin/bash", "/adei/src/scripts/cron/adei_clean.cron.sh" ]
+      env: "{{ adei_pod_env | union(adei_cron_env) }}"
+      vols: "{{ adei_pod_vols }}"
+      mounts: "{{ adei_prod_mounts | union(adei_pod_mounts) }}"
+      groups: [ "adei" ]
+      enabled: false