Source code

001/**
002 * Copyright 2005-2018 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.kuali.rice.web.health;
017
018import com.amazonaws.services.s3.AmazonS3;
019import com.codahale.metrics.Counter;
020import com.codahale.metrics.Gauge;
021import com.codahale.metrics.Histogram;
022import com.codahale.metrics.Meter;
023import com.codahale.metrics.Metric;
024import com.codahale.metrics.MetricRegistry;
025import com.codahale.metrics.Timer;
026import com.codahale.metrics.health.HealthCheck;
027import com.codahale.metrics.health.HealthCheckRegistry;
028import com.codahale.metrics.jvm.BufferPoolMetricSet;
029import com.codahale.metrics.jvm.ClassLoadingGaugeSet;
030import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
031import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
032import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
033import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
034import com.fasterxml.jackson.databind.ObjectMapper;
035import org.kuali.rice.core.api.config.property.ConfigContext;
036import org.kuali.rice.core.api.resourceloader.GlobalResourceLoader;
037import org.kuali.rice.core.api.util.RiceConstants;
038import org.kuali.rice.core.framework.persistence.platform.DatabasePlatform;
039
040import javax.servlet.ServletException;
041import javax.servlet.http.HttpServlet;
042import javax.servlet.http.HttpServletRequest;
043import javax.servlet.http.HttpServletResponse;
044import javax.sql.DataSource;
045import java.io.IOException;
046import java.lang.management.ManagementFactory;
047import java.lang.management.RuntimeMXBean;
048import java.util.Map;
049
050/**
051 * Implements an endpoint for providing health information for a Kuali Rice server.
052 *
053 * @author Eric Westfall
054 */
055public class HealthServlet extends HttpServlet {
056    
057    private MetricRegistry metricRegistry;
058    private HealthCheckRegistry healthCheckRegistry;
059    private Config config;
060
061    @Override
062    public void init() throws ServletException {
063        this.metricRegistry = new MetricRegistry();
064        this.healthCheckRegistry = new HealthCheckRegistry();
065        this.config = new Config();
066
067        monitorMemoryUsage();
068        monitorThreads();
069        monitorGarbageCollection();
070        monitorBufferPools();
071        monitorClassLoading();
072        monitorFileDescriptors();
073        monitorRuntime();
074        monitorDataSources();
075        monitorAmazonS3();
076    }
077
078    @Override
079    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
080        HealthStatus status = checkHealth();
081
082        String includeDetail = req.getParameter("detail");
083        if ("true".equals(includeDetail)) {
084            if (status.isOk()) {
085                resp.setStatus(200);
086            } else {
087                resp.setStatus(503);
088            }
089
090            ObjectMapper mapper = new ObjectMapper();
091            resp.setContentType("application/json");
092            mapper.writeValue(resp.getOutputStream(), status);
093
094        } else {
095            if (status.isOk()) {
096                resp.setStatus(204);
097            } else {
098                resp.setStatus(503);
099            }
100        }
101        resp.getOutputStream().flush();
102    }
103
104    @SuppressWarnings("unchecked")
105    private void monitorMemoryUsage() {
106        // registry memory metrics, we are going to rename this slightly using our format given the format required
107        // by the health detail specification
108        MemoryUsageGaugeSet gaugeSet = new MemoryUsageGaugeSet();
109        Map<String, Metric> metrics = gaugeSet.getMetrics();
110        for (String metricName : metrics.keySet()) {
111            this.metricRegistry.register("memory:" + metricName, metrics.get(metricName));
112        }
113
114        Gauge<Double> heapUsage = this.metricRegistry.getGauges().get("memory:heap.usage");
115        Gauge<Long> heapMaxMemory = this.metricRegistry.getGauges().get("memory:heap.max");
116        if (heapMaxMemory.getValue() != -1) {
117            this.healthCheckRegistry.register("memory:heap.usage", new MemoryUsageHealthCheck(heapUsage, config.heapMemoryUsageThreshold()));
118        }
119
120        Gauge<Double> nonHeapUsage = this.metricRegistry.getGauges().get("memory:non-heap.usage");
121        Gauge<Long> nonHeapMaxMemory = this.metricRegistry.getGauges().get("memory:non-heap.max");
122        if (nonHeapMaxMemory.getValue() != -1) {
123            this.healthCheckRegistry.register("memory:non-heap.usage", new MemoryUsageHealthCheck(nonHeapUsage, config.nonHeapMemoryUsageThreshold()));
124        }
125
126        Gauge<Long> totalUsedMemory = this.metricRegistry.getGauges().get("memory:total.used");
127        Gauge<Long> totalMaxMemory = this.metricRegistry.getGauges().get("memory:total.max");
128        if (totalMaxMemory.getValue() != -1) {
129            MemoryUsageRatio totalMemoryRatio = new MemoryUsageRatio(totalUsedMemory, totalMaxMemory);
130            this.metricRegistry.register("memory:total.usage", totalMemoryRatio);
131            this.healthCheckRegistry.register("memory:total.usage", new MemoryUsageHealthCheck(totalMemoryRatio, config.totalMemoryUsageThreshold()));
132        }
133    }
134
135    @SuppressWarnings("unchecked")
136    private void monitorThreads() {
137        ThreadStatesGaugeSet gaugeSet = new ThreadStatesGaugeSet();
138        Map<String, Metric> metrics = gaugeSet.getMetrics();
139        for (String name : metrics.keySet()) {
140            this.metricRegistry.register("thread:" + name, metrics.get(name));
141        }
142
143        // register health check for deadlock count
144        String deadlockCountName = "thread:deadlock.count";
145        final Gauge<Integer> deadlockCount = this.metricRegistry.getGauges().get(deadlockCountName);
146        this.healthCheckRegistry.register("thread:deadlock.count", new HealthCheck() {
147            @Override
148            protected Result check() throws Exception {
149                int numDeadlocks = deadlockCount.getValue();
150                if (numDeadlocks >= config.deadlockThreshold()) {
151                    return Result.unhealthy("There are " + numDeadlocks + " deadlocked threads which is greater than or equal to the threshold of " + config.deadlockThreshold());
152                }
153                return Result.healthy();
154            }
155        });
156    }
157
158    private void monitorGarbageCollection() {
159        GarbageCollectorMetricSet metricSet = new GarbageCollectorMetricSet();
160        Map<String, Metric> metrics = metricSet.getMetrics();
161        for (String name : metrics.keySet()) {
162            this.metricRegistry.register("garbage-collector:" + name, metrics.get(name));
163        }
164    }
165
166    private void monitorBufferPools() {
167        BufferPoolMetricSet metricSet = new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer());
168        Map<String, Metric> metrics = metricSet.getMetrics();
169        for (String name : metrics.keySet()) {
170            this.metricRegistry.register("buffer-pool:" + name, metrics.get(name));
171        }
172
173    }
174
175    private void monitorClassLoading() {
176        ClassLoadingGaugeSet metricSet = new ClassLoadingGaugeSet();
177        Map<String, Metric> metrics = metricSet.getMetrics();
178        for (String name : metrics.keySet()) {
179            this.metricRegistry.register("classloader:" + name, metrics.get(name));
180        }
181    }
182
183    private void monitorFileDescriptors() {
184        final FileDescriptorRatioGauge gauge = new FileDescriptorRatioGauge();
185        String name = "file-descriptor:usage";
186        this.metricRegistry.register(name, gauge);
187        this.healthCheckRegistry.register(name, new HealthCheck() {
188            @Override
189            protected Result check() throws Exception {
190                double value = gauge.getValue();
191                if (value >= config.fileDescriptorUsageThreshold()) {
192                    return Result.unhealthy("File descriptor usage ratio of " + value + " was greater than or equal to threshold of " + config.fileDescriptorUsageThreshold());
193                }
194                return Result.healthy();
195            }
196        });
197    }
198
199    private void monitorRuntime() {
200        final RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
201        this.metricRegistry.register("runtime:uptime", new Gauge<Long>() {
202            @Override
203            public Long getValue() {
204                return runtime.getUptime();
205            }
206        });
207    }
208
209    private void monitorDataSources() {
210        DataSource dataSource = (DataSource)ConfigContext.getCurrentContextConfig().getObject(RiceConstants.DATASOURCE_OBJ);
211        DataSource nonTransactionalDataSource = (DataSource)ConfigContext.getCurrentContextConfig().getObject(RiceConstants.NON_TRANSACTIONAL_DATASOURCE_OBJ);
212        DataSource serverDataSource = (DataSource)ConfigContext.getCurrentContextConfig().getObject(RiceConstants.SERVER_DATASOURCE_OBJ);
213        DatabasePlatform databasePlatform = GlobalResourceLoader.getService(RiceConstants.DB_PLATFORM);
214        monitorDataSource("database.primary:", dataSource, databasePlatform, config.primaryConnectionPoolUsageThreshold());
215        monitorDataSource("database.non-transactional:", nonTransactionalDataSource, databasePlatform, config.nonTransactionalConnectionPoolUsageThreshold());
216        monitorDataSource("database.server:", serverDataSource, databasePlatform, config.serverConnectionPoolUsageThreshold());
217    }
218
219    @SuppressWarnings("unchecked")
220    private void monitorDataSource(String namePrefix, DataSource dataSource, DatabasePlatform databasePlatform, double threshold) {
221        if (databasePlatform != null && dataSource != null) {
222            // register connection metric
223            String name = namePrefix + "connected";
224            DatabaseConnectionHealthGauge healthGauge = new DatabaseConnectionHealthGauge(dataSource, databasePlatform);
225            this.metricRegistry.register(name, healthGauge);
226            this.healthCheckRegistry.register(name, healthGauge);
227
228            // register pool metrics
229            String poolUsageName = namePrefix + DatabaseConnectionPoolMetricSet.USAGE;
230            DatabaseConnectionPoolMetricSet poolMetrics = new DatabaseConnectionPoolMetricSet(namePrefix, dataSource);
231            this.metricRegistry.registerAll(poolMetrics);
232            Gauge<Double> poolUsage = this.metricRegistry.getGauges().get(poolUsageName);
233            if (poolUsage != null) {
234                this.healthCheckRegistry.register(poolUsageName, new DatabaseConnectionPoolHealthCheck(poolUsage, threshold));
235            }
236        }
237    }
238
239    private void monitorAmazonS3() {
240        // AmazonS3 may or may not be enabled, we will check
241        AmazonS3 amazonS3 = GlobalResourceLoader.getService("amazonS3");
242        if (amazonS3 != null) {
243            AmazonS3ConnectionHealthGauge gauge = new AmazonS3ConnectionHealthGauge(amazonS3);
244            String name = "amazonS3:connected";
245            this.metricRegistry.register(name, gauge);
246            this.healthCheckRegistry.register(name, gauge);
247        }
248    }
249
250    private HealthStatus checkHealth() {
251        HealthStatus status = new HealthStatus();
252        runHealthChecks(status);
253        reportMetrics(status);
254        return status;
255    }
256
257    private void runHealthChecks(HealthStatus status) {
258        Map<String, HealthCheck.Result> results = this.healthCheckRegistry.runHealthChecks();
259        for (String name : results.keySet()) {
260            HealthCheck.Result result = results.get(name);
261            if (!result.isHealthy()) {
262                status.setStatusCode(HealthStatus.FAILED);
263                status.appendMessage(name, result.getMessage());
264            }
265        }
266    }
267
268    private void reportMetrics(HealthStatus status) {
269        reportGauges(this.metricRegistry.getGauges(), status);
270        reportCounters(metricRegistry.getCounters(), status);
271        reportHistograms(metricRegistry.getHistograms(), status);
272        reportMeters(metricRegistry.getMeters(), status);
273        reportTimers(metricRegistry.getTimers(), status);
274    }
275
276    private void reportGauges(Map<String, Gauge> gaugues, HealthStatus status) {
277        for (String name : gaugues.keySet()) {
278            Gauge gauge = gaugues.get(name);
279            status.getMetrics().add(new HealthMetric(name, gauge.getValue()));
280        }
281    }
282
283    private void reportCounters(Map<String, Counter> counters, HealthStatus status) {
284        for (String name : counters.keySet()) {
285            Counter counter = counters.get(name);
286            status.getMetrics().add(new HealthMetric(name, counter.getCount()));
287        }
288    }
289
290    private void reportHistograms(Map<String, Histogram> histograms, HealthStatus status) {
291        for (String name : histograms.keySet()) {
292            Histogram histogram = histograms.get(name);
293            status.getMetrics().add(new HealthMetric(name, histogram.getCount()));
294        }
295    }
296
297    private void reportMeters(Map<String, Meter> meters, HealthStatus status) {
298        for (String name : meters.keySet()) {
299            Meter meter = meters.get(name);
300            status.getMetrics().add(new HealthMetric(name, meter.getCount()));
301        }
302    }
303
304    private void reportTimers(Map<String, Timer> timers, HealthStatus status) {
305        for (String name : timers.keySet()) {
306            Timer timer = timers.get(name);
307            status.getMetrics().add(new HealthMetric(name, timer.getCount()));
308        }
309    }
310    
311    public static final class Config {
312
313        public static final String HEAP_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.heap.usageThreshold";
314        public static final String NON_HEAP_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.nonHeap.usageThreshold";
315        public static final String TOTAL_MEMORY_THRESHOLD_PROPERTY = "rice.health.memory.total.usageThreshold";
316        public static final String DEADLOCK_THRESHOLD_PROPERTY = "rice.health.thread.deadlockThreshold";
317        public static final String FILE_DESCRIPTOR_THRESHOLD_PROPERTY = "rice.health.fileDescriptor.usageThreshold";
318        public static final String PRIMARY_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.primary.connectionPoolUsageThreshold";
319        public static final String NON_TRANSACTIONAL_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.nonTransactional.connectionPoolUsageThreshold";
320        public static final String SERVER_POOL_USAGE_THRESHOLD_PROPERTY = "rice.health.database.server.connectionPoolUsageThreshold";
321
322        private static final double HEAP_MEMORY_THRESHOLD_DEFAULT = 0.95;
323        private static final double NON_HEAP_MEMORY_THRESHOLD_DEFAULT = 0.95;
324        private static final double TOTAL_MEMORY_THRESHOLD_DEFAULT = 0.95;
325        private static final int DEADLOCK_THRESHOLD_DEFAULT = 1;
326        private static final double FILE_DESCRIPTOR_THRESHOLD_DEFAULT = 0.95;
327        private static final double POOL_USAGE_THRESHOLD_DEFAULT = 1.0;
328
329
330        double heapMemoryUsageThreshold() {
331            return getDouble(HEAP_MEMORY_THRESHOLD_PROPERTY, HEAP_MEMORY_THRESHOLD_DEFAULT);
332        }
333
334        double nonHeapMemoryUsageThreshold() {
335            return getDouble(NON_HEAP_MEMORY_THRESHOLD_PROPERTY, NON_HEAP_MEMORY_THRESHOLD_DEFAULT);
336        }
337
338        double totalMemoryUsageThreshold() {
339            return getDouble(TOTAL_MEMORY_THRESHOLD_PROPERTY, TOTAL_MEMORY_THRESHOLD_DEFAULT);
340        }
341
342        int deadlockThreshold() {
343            return getInt(DEADLOCK_THRESHOLD_PROPERTY, DEADLOCK_THRESHOLD_DEFAULT);
344        }
345
346        double fileDescriptorUsageThreshold() {
347            return getDouble(FILE_DESCRIPTOR_THRESHOLD_PROPERTY, FILE_DESCRIPTOR_THRESHOLD_DEFAULT);
348        }
349
350        double primaryConnectionPoolUsageThreshold() {
351            return getDouble(PRIMARY_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
352        }
353
354        double nonTransactionalConnectionPoolUsageThreshold() {
355            return getDouble(NON_TRANSACTIONAL_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
356        }
357
358        double serverConnectionPoolUsageThreshold() {
359            return getDouble(SERVER_POOL_USAGE_THRESHOLD_PROPERTY, POOL_USAGE_THRESHOLD_DEFAULT);
360        }
361
362        private double getDouble(String propertyName, double defaultValue) {
363            String propertyValue = ConfigContext.getCurrentContextConfig().getProperty(propertyName);
364            if (propertyValue != null) {
365                return Double.parseDouble(propertyValue);
366            }
367            return defaultValue;
368        }
369
370        private int getInt(String propertyName, int defaultValue) {
371            String propertyValue = ConfigContext.getCurrentContextConfig().getProperty(propertyName);
372            if (propertyValue != null) {
373                return Integer.parseInt(propertyValue);
374            }
375            return defaultValue;
376        }
377        
378    }
379
380}