use statsd for indexer monitoring

opengrok-indexer/pom.xml

@@ -210,6 +210,11 @@ Portions Copyright (c) 2020-2020, Lubos Kosco <[email protected]>.
             <artifactId>micrometer-registry-prometheus</artifactId>
             <version>${micrometer.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-statsd</artifactId>
+            <version>${micrometer.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

opengrok-indexer/src/main/java/org/opengrok/indexer/Metrics.java

@@ -22,31 +22,121 @@
  */
 package org.opengrok.indexer;
 
+import io.micrometer.core.instrument.Clock;
+import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
 import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
 import io.micrometer.prometheus.PrometheusConfig;
 import io.micrometer.prometheus.PrometheusMeterRegistry;
+import io.micrometer.statsd.StatsdConfig;
+import io.micrometer.statsd.StatsdMeterRegistry;
+import io.micrometer.statsd.StatsdFlavor;
+import org.opengrok.indexer.configuration.RuntimeEnvironment;
+import org.opengrok.indexer.index.Indexer;
+import org.opengrok.indexer.logger.LoggerFactory;
 
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+/**
+ * Encapsulates logic of meter registry setup and handling.
+ * Generally, the web application publishes metrics to Prometheus and the Indexer to StatsD.
+ */
 public final class Metrics {
 
-    private static final PrometheusMeterRegistry registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+    private static final Logger LOGGER = LoggerFactory.getLogger(Metrics.class);
+
+    private static final StatsdConfig statsdConfig = new StatsdConfig() {
+        @Override
+        public String get(String k) {
+            return null;
+        }
+
+        @Override
+        public StatsdFlavor flavor() {
+            return RuntimeEnvironment.getInstance().getStatsdConfig().getFlavor();
+        }
+
+        @Override
+        public int port() {
+            return RuntimeEnvironment.getInstance().getStatsdConfig().getPort();
+        }
+
+        @Override
+        public String host() {
+            return RuntimeEnvironment.getInstance().getStatsdConfig().getHost();
+        }
+
+        @Override
+        public boolean buffered() {
+            return true;
+        }
+    };
+
+    private static PrometheusMeterRegistry prometheusRegistry;
+    private static StatsdMeterRegistry statsdRegistry;
 
     static {
-        new ClassLoaderMetrics().bindTo(registry);
-        new JvmMemoryMetrics().bindTo(registry);
-        new JvmGcMetrics().bindTo(registry);
-        new ProcessorMetrics().bindTo(registry);
-        new JvmThreadMetrics().bindTo(registry);
+        MeterRegistry registry = null;
+
+        if (RuntimeEnvironment.getInstance().getStatsdConfig().isEnabled()) {
+            LOGGER.log(Level.INFO, "configuring StatsdRegistry");
+            statsdRegistry = new StatsdMeterRegistry(statsdConfig, Clock.SYSTEM);
+            registry = statsdRegistry;
+        } else if (!RuntimeEnvironment.getInstance().isIndexer()) {
+            LOGGER.log(Level.INFO, "configuring PrometheusRegistry");
+            prometheusRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
+            registry = prometheusRegistry;
+        }
+
+        if (registry != null) {
+            new ClassLoaderMetrics().bindTo(registry);
+            new JvmMemoryMetrics().bindTo(registry);
+            new JvmGcMetrics().bindTo(registry);
+            new ProcessorMetrics().bindTo(registry);
+            new JvmThreadMetrics().bindTo(registry);
+        }
     }
 
     private Metrics() {
     }
 
-    public static PrometheusMeterRegistry getRegistry() {
-        return registry;
+    public static void updateSubFiles(List<String> subFiles) {
+        // Add tag for per-project reindex.
+        if (statsdRegistry != null && !subFiles.isEmpty()) {
+            String projects = subFiles.stream().
+                    map(s -> s.startsWith(Indexer.PATH_SEPARATOR_STRING) ? s.substring(1) : s).
+                    collect(Collectors.joining(","));
+            Tag commonTag = Tag.of("projects", projects);
+            LOGGER.log(Level.FINE, "updating statsdRegistry with common tag: {}", commonTag);
+            statsdRegistry.config().commonTags(Collections.singleton(commonTag));
+        }
+    }
+
+    public static PrometheusMeterRegistry getPrometheusRegistry() {
+        return prometheusRegistry;
     }
 
+    private static StatsdMeterRegistry getStatsdRegistry() {
+        return statsdRegistry;
+    }
+
+    /**
+     * Get registry based on running context.
+     * @return MeterRegistry instance
+     */
+    public static MeterRegistry getRegistry() {
+        if (RuntimeEnvironment.getInstance().isIndexer()) {
+            return getStatsdRegistry();
+        } else {
+            return getPrometheusRegistry();
+        }
+    }
 }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -299,6 +299,8 @@ public final class Configuration {
 
     private SuggesterConfig suggesterConfig = new SuggesterConfig();
 
+    private StatsdConfig statsdConfig = new StatsdConfig();
+
     private Set<String> disabledRepositories;
 
     /*
@@ -1315,6 +1317,17 @@ public void setSuggesterConfig(final SuggesterConfig config) {
         this.suggesterConfig = config;
     }
 
+    public StatsdConfig getStatsdConfig() {
+        return statsdConfig;
+    }
+
+    public void setStatsdConfig(final StatsdConfig config) {
+        if (config == null) {
+            throw new IllegalArgumentException("Cannot set Statsd configuration to null");
+        }
+        this.statsdConfig = config;
+    }
+
     public Set<String> getDisabledRepositories() {
         return disabledRepositories;
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/ConfigurationHelp.java

@@ -183,6 +183,8 @@ private static Object getSampleValue(Method setter, Object defaultValue) {
             return null;
         } else if (paramType == SuggesterConfig.class) {
             return SuggesterConfig.getForHelp();
+        } else if (paramType == StatsdConfig.class) {
+            return StatsdConfig.getForHelp();
         } else {
             throw new UnsupportedOperationException("getSampleValue() for " +
                 paramType + ", " + genType);

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java

@@ -123,6 +123,12 @@ public final class RuntimeEnvironment {
 
     public WatchDogService watchDog;
 
+    public List<String> getSubFiles() {
+        return subFiles;
+    }
+
+    private List<String> subFiles = new ArrayList<>();
+
     /**
      * Creates a new instance of RuntimeEnvironment. Private to ensure a
      * singleton anti-pattern.
@@ -141,6 +147,16 @@ private RuntimeEnvironment() {
     private AuthorizationFramework authFramework;
     private final Object authFrameworkLock = new Object();
 
+    private boolean indexer;
+
+    public boolean isIndexer() {
+        return indexer;
+    }
+
+    public void setIndexer(boolean indexer) {
+        this.indexer = indexer;
+    }
+
     /** Gets the thread pool used for multi-project searches. */
     public ExecutorService getSearchExecutor() {
         return lzSearchExecutor.get();
@@ -1876,6 +1892,14 @@ public void setSuggesterConfig(SuggesterConfig suggesterConfig) {
         syncWriteConfiguration(suggesterConfig, Configuration::setSuggesterConfig);
     }
 
+    public StatsdConfig getStatsdConfig() {
+        return syncReadConfiguration(Configuration::getStatsdConfig);
+    }
+
+    public void setStatsdConfig(StatsdConfig statsdConfig) {
+        syncWriteConfiguration(statsdConfig, Configuration::setStatsdConfig);
+    }
+
     /**
      * Applies the specified function to the runtime configuration, after having
      * obtained the configuration read-lock (and releasing afterward).

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/StatsdConfig.java

@@ -0,0 +1,76 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * See LICENSE.txt included in this distribution for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at LICENSE.txt.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
+ */
+
+package org.opengrok.indexer.configuration;
+
+import io.micrometer.statsd.StatsdFlavor;
+
+/**
+ * Configuration for Statsd metrics emitted by the Indexer via {@link org.opengrok.indexer.util.Statistics}.
+ */
+public class StatsdConfig {
+    private int port;
+    private String host;
+    private boolean enabled;
+    private StatsdFlavor flavor;
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public void setPort(int port) {
+        this.port = port;
+    }
+
+    public StatsdFlavor getFlavor() {
+        return flavor;
+    }
+
+    public void setFlavor(StatsdFlavor flavor) {
+        this.flavor = flavor;
+    }
+
+    public boolean isEnabled() {
+        return port != 0 && host != null && !host.isEmpty() && flavor != null;
+    }
+
+    /**
+     * Gets an instance version suitable for helper documentation by shifting
+     * most default properties slightly.
+     */
+    static StatsdConfig getForHelp() {
+        StatsdConfig res = new StatsdConfig();
+        res.setHost("foo.bar");
+        res.setPort(8125);
+        res.setFlavor(StatsdFlavor.ETSY);
+        return res;
+    }
+}

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -618,7 +618,7 @@ public void run() {
             LOGGER.log(Level.WARNING,
                     "Failed optimizing the history cache database", he);
         }
-        elapsed.report(LOGGER, "Done historycache for all repositories");
+        elapsed.report(LOGGER, "Done history cache for all repositories", "indexer.history.cache");
         historyCache.setHistoryIndexDone();
     }
 
@@ -884,7 +884,8 @@ public void run() {
         repositories.clear();
         newrepos.forEach((_key, repo) -> putRepository(repo));
 
-        elapsed.report(LOGGER, String.format("done invalidating %d repositories", newrepos.size()));
+        elapsed.report(LOGGER, String.format("done invalidating %d repositories", newrepos.size()),
+                "history.repositories.invalidate");
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java

@@ -50,7 +50,9 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+
 import org.opengrok.indexer.Info;
+import org.opengrok.indexer.Metrics;
 import org.opengrok.indexer.analysis.AnalyzerGuru;
 import org.opengrok.indexer.analysis.AnalyzerGuruHelp;
 import org.opengrok.indexer.analysis.Ctags;
@@ -148,7 +150,7 @@ public static void main(String[] argv) {
         boolean update = true;
 
         Executor.registerErrorHandler();
-        ArrayList<String> subFiles = new ArrayList<>();
+        List<String> subFiles = RuntimeEnvironment.getInstance().getSubFiles();
         ArrayList<String> subFilesList = new ArrayList<>();
 
         boolean createDict = false;
@@ -177,6 +179,7 @@ public static void main(String[] argv) {
             }
 
             env = RuntimeEnvironment.getInstance();
+            env.setIndexer(true);
 
             // Complete the configuration of repository types.
             List<Class<? extends Repository>> repositoryClasses
@@ -316,6 +319,8 @@ public static void main(String[] argv) {
                 System.exit(1);
             }
 
+            Metrics.updateSubFiles(subFiles);
+
             // If the webapp is running with a config that does not contain
             // 'projectsEnabled' property (case of upgrade or transition
             // from project-less config to one with projects), set the property
@@ -385,7 +390,7 @@ public static void main(String[] argv) {
             System.err.println("Exception: " + e.getLocalizedMessage());
             System.exit(1);
         } finally {
-            stats.report(LOGGER);
+            stats.report(LOGGER, "Indexer finished", "indexer.total");
         }
     }
 
@@ -990,7 +995,7 @@ public void prepareIndexer(RuntimeEnvironment env,
             Statistics stats = new Statistics();
             env.setRepositories(searchPaths.toArray(new String[0]));
             stats.report(LOGGER, String.format("Done scanning for repositories, found %d repositories",
-                    env.getRepositories().size()));
+                    env.getRepositories().size()), "indexer.repository.scan");
         }
 
         if (createHistoryCache) {
@@ -1102,7 +1107,7 @@ public void run() {
             LOGGER.log(Level.WARNING, "Received interrupt while waiting" +
                     " for executor to finish", exp);
         }
-        elapsed.report(LOGGER, "Done indexing data of all repositories");
+        elapsed.report(LOGGER, "Done indexing data of all repositories", "indexer.repository.indexing");
 
         CtagsUtil.deleteTempFiles();
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/util/Executor.java

@@ -232,8 +232,9 @@ public void run() {
 
             ret = process.waitFor();
 
-            stat.report(LOGGER, Level.FINE, String.format("Finished command [%s] in directory %s with exit code %d",
-                    cmd_str, dir_str, ret));
+            stat.report(LOGGER, Level.FINE,
+                    String.format("Finished command [%s] in directory %s with exit code %d", cmd_str, dir_str, ret),
+                    "executor.latency");
             LOGGER.log(Level.FINE,
                 "Finished command [{0}] in directory {1} with exit code {2}",
                 new Object[] {cmd_str, dir_str, ret});