001 /**
002 * Copyright 2010-2013 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.kuali.common.jdbc;
017
018 import java.io.IOException;
019 import java.sql.Connection;
020 import java.sql.DatabaseMetaData;
021 import java.sql.SQLException;
022 import java.sql.Statement;
023 import java.util.ArrayList;
024 import java.util.Arrays;
025 import java.util.Collections;
026 import java.util.List;
027
028 import javax.sql.DataSource;
029
030 import org.apache.commons.lang3.StringUtils;
031 import org.kuali.common.jdbc.context.JdbcContext;
032 import org.kuali.common.jdbc.listener.BucketEvent;
033 import org.kuali.common.jdbc.listener.LogSqlListener;
034 import org.kuali.common.jdbc.listener.MultiThreadedExecutionListener;
035 import org.kuali.common.jdbc.listener.NotifyingListener;
036 import org.kuali.common.jdbc.listener.SqlEvent;
037 import org.kuali.common.jdbc.listener.SqlExecutionEvent;
038 import org.kuali.common.jdbc.listener.SqlListener;
039 import org.kuali.common.jdbc.listener.SqlMetaDataEvent;
040 import org.kuali.common.jdbc.supplier.SimpleStringSupplier;
041 import org.kuali.common.jdbc.supplier.SqlSupplier;
042 import org.kuali.common.jdbc.threads.SqlBucket;
043 import org.kuali.common.jdbc.threads.SqlBucketContext;
044 import org.kuali.common.jdbc.threads.SqlBucketHandler;
045 import org.kuali.common.threads.ExecutionStatistics;
046 import org.kuali.common.threads.ThreadHandlerContext;
047 import org.kuali.common.threads.ThreadInvoker;
048 import org.kuali.common.util.CollectionUtils;
049 import org.kuali.common.util.FormatUtils;
050 import org.kuali.common.util.PercentCompleteInformer;
051 import org.kuali.common.util.Str;
052 import org.slf4j.Logger;
053 import org.slf4j.LoggerFactory;
054 import org.springframework.jdbc.datasource.DataSourceUtils;
055
056 /**
057 * @deprecated
058 */
059 @Deprecated
060 public class DefaultJdbcService implements JdbcService {
061
062 private static final Logger logger = LoggerFactory.getLogger(DefaultJdbcService.class);
063
064 @Override
065 public ExecutionResult executeSql(JdbcContext context) {
066 ExecutionStats stats = new ExecutionStats();
067 long start = System.currentTimeMillis();
068
069 // Log a message if provided
070 if (!StringUtils.isBlank(context.getMessage())) {
071 logger.info(context.getMessage());
072 }
073
074 // Make sure we have something to do
075 if (CollectionUtils.isEmpty(context.getSuppliers())) {
076 logger.info("Skipping execution. No suppliers");
077 return new ExecutionResult(0, start, System.currentTimeMillis(), 0);
078 }
079
080 // Calculate metadata
081 if (!context.isSkipMetaData()) {
082 doMetaData(context);
083 }
084
085 // Fire an event before executing any SQL
086 long sqlStart = System.currentTimeMillis();
087 context.getListener().beforeExecution(new SqlExecutionEvent(context, start, -1));
088
089 // Execute the SQL as dictated by the context
090 if (context.isMultithreaded()) {
091 stats = executeMultiThreaded(context);
092 } else {
093 stats = executeSequentially(context);
094 }
095
096 // Fire an event now that all SQL execution is complete
097 context.getListener().afterExecution(new SqlExecutionEvent(context, sqlStart, System.currentTimeMillis()));
098
099 return new ExecutionResult(stats.getUpdateCount(), start, System.currentTimeMillis(), stats.getStatementCount());
100 }
101
102 protected void doMetaData(JdbcContext context) {
103
104 logger.debug("doMetaData()");
105
106 // Fire an event before we begin calculating metadata
107 long start = System.currentTimeMillis();
108 context.getListener().beforeMetaData(new SqlMetaDataEvent(context, start, -1));
109
110 // Fill in SQL metadata
111 for (SqlSupplier supplier : context.getSuppliers()) {
112 supplier.fillInMetaData();
113 }
114
115 // Fire an event now that metadata calculation is complete
116 context.getListener().afterMetaData(new SqlMetaDataEvent(context, start, System.currentTimeMillis()));
117 }
118
119 protected ExecutionStats executeMultiThreaded(JdbcContext context) {
120
121 // Divide the SQL we have to execute up into buckets as "evenly" as possible
122 List<SqlBucket> buckets = getSqlBuckets(context);
123
124 // Notify the listener now that buckets are created
125 context.getListener().bucketsCreated(new BucketEvent(context, buckets));
126
127 // Sort the buckets largest to smallest
128 Collections.sort(buckets);
129 Collections.reverse(buckets);
130
131 // The tracking built into the kuali-threads package assumes "progress" equals one element from the list completing
132 // It assumes you have a gigantic list where each element in the list = 1 unit of work
133 // A large list of files that need to be posted to S3 (for example).
134 // If we could randomly split up the SQL and execute it in whatever order we wanted, the built in tracking would work.
135 // We cannot do that though, since the SQL in each file needs to execute sequentially in order
136 // SQL from different files can execute concurrently, but the SQL inside each file needs to execute in order
137 // For example OLE has ~250,000 SQL statements split up across ~300 files
138 // In addition, the schema related DDL files need to execute first, then data, then constraints DDL files
139 // Some files are HUGE, some are tiny. Printing a dot after each file completes doesn't make sense.
140 // Our list of buckets is pretty small, even though the total number of SQL statements is quite large
141 // Only printing a dot to the console when each bucket completes is not granular enough
142
143 // This listener prints a dot each time 1% of the total number of SQL statements across all of the buckets has been executed.
144 long total = JdbcUtils.getSqlCount(context.getSuppliers());
145 PercentCompleteInformer informer = new PercentCompleteInformer(total);
146 MultiThreadedExecutionListener etl = new MultiThreadedExecutionListener();
147 etl.setTrackProgressByUpdateCount(context.isTrackProgressByUpdateCount());
148 etl.setInformer(informer);
149 List<SqlListener> listeners = new ArrayList<SqlListener>();
150 listeners.add(new LogSqlListener());
151 listeners.add(etl);
152 NotifyingListener nl = new NotifyingListener(listeners);
153
154 // Provide some context for each bucket
155 List<SqlBucketContext> sbcs = getSqlBucketContexts(buckets, context, nl);
156
157 // Store some context for the thread handler
158 ThreadHandlerContext<SqlBucketContext> thc = new ThreadHandlerContext<SqlBucketContext>();
159 thc.setList(sbcs);
160 thc.setHandler(new SqlBucketHandler());
161 thc.setMax(buckets.size());
162 thc.setMin(buckets.size());
163 thc.setDivisor(1);
164
165 // Start threads to execute SQL from multiple suppliers concurrently
166 ThreadInvoker invoker = new ThreadInvoker();
167 informer.start();
168 ExecutionStatistics stats = invoker.invokeThreads(thc);
169 informer.stop();
170
171 // Display thread related stats
172 long aggregateTime = etl.getAggregateTime();
173 long wallTime = stats.getExecutionTime();
174 String avgMillis = FormatUtils.getTime(aggregateTime / buckets.size());
175 String aTime = FormatUtils.getTime(aggregateTime);
176 String wTime = FormatUtils.getTime(wallTime);
177 String sqlCount = FormatUtils.getCount(etl.getAggregateSqlCount());
178 String sqlSize = FormatUtils.getSize(etl.getAggregateSqlSize());
179 Object[] args = { buckets.size(), wTime, aTime, avgMillis, sqlCount, sqlSize };
180 logger.debug("Threads - [count: {} time: {} aggregate: {} avg: {} sql: {} - {}]", args);
181
182 return new ExecutionStats(etl.getAggregateUpdateCount(), etl.getAggregateSqlCount());
183 }
184
185 @Override
186 public ExecutionResult executeSql(DataSource dataSource, String sql) {
187 return executeSql(dataSource, Arrays.asList(sql));
188 }
189
190 @Override
191 public ExecutionResult executeSql(DataSource dataSource, List<String> sql) {
192 SqlSupplier supplier = new SimpleStringSupplier(sql);
193 JdbcContext context = new JdbcContext();
194 context.setDataSource(dataSource);
195 context.setSuppliers(Arrays.asList(supplier));
196 return executeSql(context);
197 }
198
199 protected List<SqlBucketContext> getSqlBucketContexts(List<SqlBucket> buckets, JdbcContext context, SqlListener listener) {
200 List<SqlBucketContext> sbcs = new ArrayList<SqlBucketContext>();
201
202 for (SqlBucket bucket : buckets) {
203
204 JdbcContext newJdbcContext = getJdbcContext(context, bucket, listener);
205
206 SqlBucketContext sbc = new SqlBucketContext();
207 sbc.setService(this);
208 sbc.setBucket(bucket);
209 sbc.setContext(newJdbcContext);
210 sbcs.add(sbc);
211 }
212 return sbcs;
213 }
214
215 protected JdbcContext getJdbcContext(JdbcContext original, SqlBucket bucket, SqlListener listener) {
216 JdbcContext context = new JdbcContext();
217 context.setSuppliers(bucket.getSuppliers());
218 context.setDataSource(original.getDataSource());
219 context.setCommitMode(original.getCommitMode());
220 context.setThreads(1);
221 context.setSkip(original.isSkip());
222 context.setListener(listener);
223 context.setSkipMetaData(true);
224 return context;
225 }
226
227 protected List<SqlBucket> getSqlBuckets(JdbcContext context) {
228
229 // Pull out our list of suppliers
230 List<SqlSupplier> suppliers = context.getSuppliers();
231
232 // number of buckets equals thread count, unless thread count > total number of sources
233 int bucketCount = Math.min(context.getThreads(), suppliers.size());
234
235 // Sort the suppliers by SQL size
236 Collections.sort(suppliers);
237
238 // Largest to smallest instead of smallest to largest
239 Collections.reverse(suppliers);
240
241 // Allocate some buckets to hold the sql
242 List<SqlBucket> buckets = CollectionUtils.getNewList(SqlBucket.class, bucketCount);
243
244 // Distribute the sources into buckets as evenly as possible
245 // "Evenly" in this case means each bucket should be roughly the same size
246 for (SqlSupplier supplier : suppliers) {
247
248 // Sort the buckets by size
249 Collections.sort(buckets);
250
251 // First bucket in the list is the smallest
252 SqlBucket smallest = buckets.get(0);
253
254 // Add this source to the bucket
255 smallest.getSuppliers().add(supplier);
256
257 // Update the bucket metadata holding overall size
258 smallest.setCount(smallest.getCount() + supplier.getMetaData().getCount());
259 smallest.setSize(smallest.getSize() + supplier.getMetaData().getSize());
260 }
261
262 // Return the buckets
263 return buckets;
264 }
265
266 protected ExecutionStats executeSequentially(JdbcContext context) {
267 Connection conn = null;
268 Statement statement = null;
269 try {
270 long updateCount = 0;
271 long statementCount = 0;
272 conn = DataSourceUtils.doGetConnection(context.getDataSource());
273 boolean originalAutoCommitSetting = conn.getAutoCommit();
274 conn.setAutoCommit(false);
275 statement = conn.createStatement();
276 List<SqlSupplier> suppliers = context.getSuppliers();
277 for (SqlSupplier supplier : suppliers) {
278 ExecutionStats stats = excecuteSupplier(statement, context, supplier);
279 updateCount += stats.getUpdateCount();
280 statementCount += stats.getStatementCount();
281 conn.commit();
282 }
283 conn.setAutoCommit(originalAutoCommitSetting);
284 return new ExecutionStats(updateCount, statementCount);
285 } catch (Exception e) {
286 throw new IllegalStateException(e);
287 } finally {
288 JdbcUtils.closeQuietly(context.getDataSource(), conn, statement);
289 }
290 }
291
292 protected ExecutionStats excecuteSupplier(Statement statement, JdbcContext context, SqlSupplier supplier) throws SQLException {
293 try {
294 long updateCount = 0;
295 long statementCount = 0;
296 supplier.open();
297 List<String> sql = supplier.getSql();
298 while (sql != null) {
299 for (String s : sql) {
300 updateCount += executeSql(statement, s, context);
301 statementCount++;
302 }
303 sql = supplier.getSql();
304 }
305 return new ExecutionStats(updateCount, statementCount);
306 } catch (IOException e) {
307 throw new IllegalStateException(e);
308 } finally {
309 supplier.close();
310 }
311 }
312
313 protected int executeSql(Statement statement, String sql, JdbcContext context) throws SQLException {
314 try {
315 int updateCount = 0;
316 long start = System.currentTimeMillis();
317 context.getListener().beforeExecuteSql(new SqlEvent(sql, start));
318 if (!context.isSkip()) {
319
320 // Execute the SQL
321 statement.execute(sql);
322
323 // Get the number of rows updated as a result of executing this SQL
324 updateCount = statement.getUpdateCount();
325
326 // Some SQL statements have nothing to do with updating rows
327 updateCount = (updateCount == -1) ? 0 : updateCount;
328 }
329 context.getListener().afterExecuteSql(new SqlEvent(sql, updateCount, start, System.currentTimeMillis()));
330 return updateCount;
331 } catch (SQLException e) {
332 throw new SQLException("Error executing SQL [" + Str.flatten(sql) + "]", e);
333 }
334 }
335
336 @Override
337 public JdbcMetaData getJdbcMetaData(DataSource dataSource) {
338 Connection conn = null;
339 try {
340 conn = DataSourceUtils.doGetConnection(dataSource);
341 DatabaseMetaData dbmd = conn.getMetaData();
342 return getJdbcMetaData(dbmd);
343 } catch (Exception e) {
344 throw new IllegalStateException(e);
345 } finally {
346 logger.trace("closing connection");
347 JdbcUtils.closeQuietly(dataSource, conn);
348 }
349 }
350
351 protected JdbcMetaData getJdbcMetaData(DatabaseMetaData dbmd) throws SQLException {
352 JdbcMetaData md = new JdbcMetaData();
353 md.setDatabaseProductName(dbmd.getDatabaseProductName());
354 md.setDatabaseProductVersion(dbmd.getDatabaseProductVersion());
355 md.setDriverName(dbmd.getDriverName());
356 md.setDriverVersion(dbmd.getDriverVersion());
357 md.setUrl(dbmd.getURL());
358 md.setUsername(dbmd.getUserName());
359 return md;
360 }
361 }