diff --git a/grails-data-mongodb/core/build.gradle b/grails-data-mongodb/core/build.gradle index 0ce227148fe..1e17eeda811 100644 --- a/grails-data-mongodb/core/build.gradle +++ b/grails-data-mongodb/core/build.gradle @@ -149,7 +149,11 @@ dependencies { // test: GenericWebApplicationContext requires ServletContext on the classpath } - testImplementation 'org.slf4j:slf4j-nop' // Prevents warning about missing slf4j implementation during compilation and tests + testImplementation 'ch.qos.logback:logback-classic', { + // test: a real SLF4J binding, so that the log a test asserts on is actually emitted. + // Quietened to WARN by src/test/resources/logback-test.xml; a test raises the level of the + // one logger it is interested in. + } testImplementation 'org.testcontainers:testcontainers' testImplementation 'org.testcontainers:testcontainers-mongodb' testImplementation 'org.testcontainers:testcontainers-spock' diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java index 1be229be4e9..c5478a29605 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/MongoDatastore.java @@ -25,6 +25,9 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import groovy.lang.Closure; @@ -162,6 +165,15 @@ public class MongoDatastore extends AbstractDatastore implements MappingContext. protected final boolean stateless; protected final boolean codecEngine; protected final boolean transactionsEnabled; + protected final boolean buildIndexes; + protected final boolean buildIndexesAsync; + + /** + * Runs the startup index build off the thread that creates the datastore when + * {@code grails.mongodb.buildIndexesAsync} is enabled; {@code null} otherwise. A single thread, + * so the indexes are still built one at a time rather than all at once against the server. + */ + private final ExecutorService indexBuildExecutor; private volatile Boolean transactionsSupported; private volatile boolean warnedTransactionsUnsupported = false; protected CodecRegistry codecRegistry; @@ -205,6 +217,11 @@ public MongoDatastore(final ConnectionSources getConnecti } /** - * Builds the MongoDB index for this datastore + * Builds the MongoDB index for this datastore. + * + *

Each index is created by a command that the server answers only once the index has been built, + * so with the default settings this blocks whoever creates the datastore — in an application, the + * startup thread — for as long as MongoDB takes to build every declared index. Enabling + * {@code grails.mongodb.buildIndexesAsync} hands the work to a background thread and returns + * immediately instead. */ public void buildIndex() { + if (!buildIndexes) { + LOG.info("Index creation is disabled by [{} = false]. The indexes declared by the domain classes " + + "will not be created or reconciled; the indexes already present on the server are left untouched.", + MongoSettings.SETTING_BUILD_INDEXES); + return; + } + if (indexBuildExecutor == null) { + buildDeclaredIndexes(); + return; + } + LOG.info("Building the indexes declared by the domain classes on a background thread ([{} = true]). " + + "Startup does not wait for them, so a query issued before its index exists is served without it.", + MongoSettings.SETTING_BUILD_INDEXES_ASYNC); + indexBuildExecutor.execute(() -> { + try { + buildDeclaredIndexes(); + } + catch (Throwable e) { + // Nothing is waiting on this thread, so an error that would have failed startup has to be + // reported here or it is lost entirely. + if (indexBuildExecutor.isShutdown() || Thread.currentThread().isInterrupted()) { + LOG.debug("The background index build was abandoned because the datastore is shutting down: {}", + e.getMessage(), e); + } + else { + LOG.error("The background index build failed: {}. The application is running without the " + + "indexes that were not created.", e.getMessage(), e); + } + } + }); + } + + /** + * Creates and reconciles the indexes declared by every entity mapped to this datastore, and reports + * what that cost. MongoDB answers each {@code createIndex} only once the index exists, so the elapsed + * time is the time the caller — startup, or the background build thread — actually spent waiting. + */ + private void buildDeclaredIndexes() { + long startedAt = System.nanoTime(); + IndexBuildSummary summary = new IndexBuildSummary(); for (PersistentEntity entity : this.mappingContext.getPersistentEntities()) { // Only create Mongo templates for entities that are mapped with Mongo if (!entity.isExternal()) { if (entity.isMultiTenant() && multiTenancyMode == MultiTenancySettings.MultiTenancyMode.SCHEMA) continue; - initializeIndices(entity); + summary.entities++; + initializeIndices(entity, summary); } } + long elapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); + if (summary.applied() == 0 && summary.failures == 0) { + LOG.debug("No indexes are declared by the {} domain class(es) mapped to database [{}]", + summary.entities, defaultDatabase); + return; + } + String outcome = summary.classified ? + summary.created + " created, " + summary.alreadyPresent + " already present" : + summary.applied() + " index declaration(s) applied"; + if (summary.failures == 0) { + LOG.info("Index build for database [{}] finished in {}ms: {}, from {} domain class(es)", + defaultDatabase, elapsedMillis, outcome, summary.entities); + } + else { + LOG.warn("Index build for database [{}] finished in {}ms: {}, {} failed, from {} domain class(es). " + + "The failures are reported above.", + defaultDatabase, elapsedMillis, outcome, summary.failures, summary.entities); + } + } + + /** + * The indexes a collection already had when the build reached it, listed once on first use and then + * reused. {@code createIndex} is idempotent and answers the same way whether or not it had to build + * anything — the driver hands back only the index name, discarding the {@code numIndexesBefore} / + * {@code numIndexesAfter} the server reports — so what was there beforehand is what distinguishes an + * index this build created from one it merely confirmed. + * + *

Listed lazily so that an entity declaring no indexes costs no round trip, and reused by the + * conflict path, which would otherwise list them again. + */ + private static final class ExistingIndexes { + + private final com.mongodb.client.MongoCollection collection; + + private final IndexBuildSummary summary; + + private List indexes; + + private boolean listed; + + private ExistingIndexes(com.mongodb.client.MongoCollection collection, IndexBuildSummary summary) { + this.collection = collection; + this.summary = summary; + } + + /** + * @return the indexes present before the build, or {@code null} if they could not be listed + */ + private List get() { + if (!listed) { + listed = true; + try { + indexes = collection.listIndexes().into(new ArrayList<>()); + } catch (RuntimeException e) { + // Not fatal: the build can still create indexes, it just cannot report which of them + // were new. Losing the breakdown is not worth failing a startup over. + LOG.debug("Could not list the existing indexes of collection [{}]: {}", + collection.getNamespace().getCollectionName(), e.getMessage(), e); + summary.classified = false; + } + } + return indexes; + } + + private boolean contains(Document keys) { + List existing = get(); + return existing != null && findIndexByKeyPattern(existing, keys) != null; + } + } + + /** + * Counts the work one index build did, so that it can be summarised once at the end rather than a line + * per index. + */ + private static final class IndexBuildSummary { + + private int entities; + + private int created; + + private int alreadyPresent; + + private int failures; + + /** + * False once an entity's existing indexes could not be listed, which is the only thing that + * separates a created index from one that was already there. The summary then falls back to + * reporting how many declarations were applied without saying which did work. + */ + private boolean classified = true; + + private int applied() { + return created + alreadyPresent; + } } /** @@ -754,6 +912,29 @@ public boolean isTransactionsEnabled() { } } + /** + * Whether GORM creates and reconciles the indexes declared in the domain class mapping blocks when + * the datastore starts. Disabled with {@code grails.mongodb.buildIndexes = false}, which leaves the + * indexes on the server exactly as they are. + * + * @return {@code true} if declared indexes are created on startup + * @since 8.0 + */ + public boolean isBuildIndexes() { + return buildIndexes; + } + + /** + * Whether the startup index build runs on a background thread instead of blocking the thread that + * creates the datastore. Enabled with {@code grails.mongodb.buildIndexesAsync = true}. + * + * @return {@code true} if declared indexes are built asynchronously + * @since 8.0 + */ + public boolean isBuildIndexesAsync() { + return buildIndexesAsync; + } + public String getDatabaseName(PersistentEntity entity) { if (entity.isMultiTenant() && multiTenancyMode == MultiTenancySettings.MultiTenancyMode.SCHEMA) { return Tenants.currentId(getClass()).toString(); @@ -921,7 +1102,17 @@ protected void registerEventListeners(ConfigurableApplicationEventPublisher even * @param entity The entity */ protected void initializeIndices(final PersistentEntity entity) { + initializeIndices(entity, new IndexBuildSummary()); + } + + private void initializeIndices(final PersistentEntity entity, final IndexBuildSummary summary) { + if (!buildIndexes) { + LOG.debug("Index creation is disabled by [{} = false]. Skipping the indexes declared by entity [{}].", + MongoSettings.SETTING_BUILD_INDEXES, entity.getName()); + return; + } final com.mongodb.client.MongoCollection collection = getCollection(entity); + final ExistingIndexes existingIndexes = new ExistingIndexes(collection, summary); final ClassMapping classMapping = entity.getMapping(); if (classMapping != null) { final MongoCollection mappedForm = classMapping.getMappedForm(); @@ -929,7 +1120,8 @@ protected void initializeIndices(final PersistentEntity entity) { List indices = mappedForm.getIndices(); for (MongoCollection.Index index : indices) { createOrUpdateIndex(entity, collection, new Document(index.getDefinition()), - index.getOptions(), "with definition [" + index.getDefinition() + "]"); + index.getOptions(), "with definition [" + index.getDefinition() + "]", + summary, existingIndexes); } for (Map compoundIndex : mappedForm.getCompoundIndices()) { @@ -943,7 +1135,7 @@ protected void initializeIndices(final PersistentEntity entity) { } Document indexDef = new Document(compoundIndex); createOrUpdateIndex(entity, collection, indexDef, indexAttributes, - "compound index with definition [" + indexDef + "]"); + "compound index with definition [" + indexDef + "]", summary, existingIndexes); } } } @@ -968,7 +1160,7 @@ protected void initializeIndices(final PersistentEntity entity) { } } createOrUpdateIndex(entity, collection, dbObject, options, - "on property [" + property.getName() + "]"); + "on property [" + property.getName() + "]", summary, existingIndexes); } } @@ -989,7 +1181,8 @@ protected void initializeIndices(final PersistentEntity entity) { */ private void createOrUpdateIndex(PersistentEntity entity, com.mongodb.client.MongoCollection collection, - Document keys, Map rawOptions, String descriptor) { + Document keys, Map rawOptions, String descriptor, + IndexBuildSummary summary, ExistingIndexes existingIndexes) { Map options = rawOptions != null ? new HashMap<>(rawOptions) : new HashMap<>(); // Control flag — not a Mongo index option. @@ -1006,13 +1199,32 @@ private void createOrUpdateIndex(PersistentEntity entity, indexOptions.expireAfter(expireAfterSeconds, TimeUnit.SECONDS); } + // Asked before the index is created, while the answer still means something. + boolean present = existingIndexes.contains(keys); + long startedAt = System.nanoTime(); try { collection.createIndex(keys, indexOptions); + if (present) { + summary.alreadyPresent++; + } + else { + summary.created++; + } + LOG.debug("{} index for entity [{}] {} in {}ms", present ? "Confirmed" : "Created", + entity.getName(), descriptor, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt)); } catch (MongoCommandException e) { if (e.getErrorCode() == INDEX_OPTIONS_CONFLICT_CODE) { - reconcileIndexConflict(entity, collection, keys, indexOptions, - expireAfterSeconds, recreateOnConflict, descriptor, e); + if (reconcileIndexConflict(entity, collection, existingIndexes, keys, indexOptions, + expireAfterSeconds, recreateOnConflict, descriptor, e)) { + // A conflict means an index was already on these keys; reconciling it changed the one + // that was there rather than adding one. + summary.alreadyPresent++; + } + else { + summary.failures++; + } } else { + summary.failures++; LOG.error("Failed to create index for entity [{}] {}: {}", entity.getName(), descriptor, e.getMessage(), e); } @@ -1024,23 +1236,26 @@ private void createOrUpdateIndex(PersistentEntity entity, * different options. A TTL difference is the common, safe case (e.g. a configurable retention * changed between restarts) and is updated in place via {@code collMod}; anything else needs an * explicit {@code recreateOnConflict:true} to authorise the drop-and-recreate. + * + * @return {@code true} if the index ended up in the declared state, {@code false} if the conflict + * could not be resolved and the existing index was left as it was */ - private void reconcileIndexConflict(PersistentEntity entity, + private boolean reconcileIndexConflict(PersistentEntity entity, com.mongodb.client.MongoCollection collection, + ExistingIndexes existingIndexes, Document keys, IndexOptions desired, Long expireAfterSeconds, boolean recreateOnConflict, String descriptor, MongoCommandException original) { - Document existing; - try { - existing = findIndexByKeyPattern(collection, keys); - } catch (RuntimeException listError) { + List indexes = existingIndexes.get(); + if (indexes == null) { LOG.error("Failed to create index for entity [{}] {} and could not inspect existing indexes: {}", - entity.getName(), descriptor, listError.getMessage(), original); - return; + entity.getName(), descriptor, original.getMessage(), original); + return false; } + Document existing = findIndexByKeyPattern(indexes, keys); if (existing == null) { LOG.error("Failed to create index for entity [{}] {}: {}", entity.getName(), descriptor, original.getMessage(), original); - return; + return false; } String existingName = existing.getString("name"); @@ -1057,7 +1272,7 @@ private void reconcileIndexConflict(PersistentEntity entity, .append(INDEX_EXPIRE_AFTER_SECONDS, expireAfterSeconds))); LOG.info("Updated TTL of index [{}] on entity [{}] to {}s", existingName, entity.getName(), expireAfterSeconds); - return; + return true; } catch (MongoCommandException collModError) { // collMod can't make every change (e.g. add a TTL to a non-TTL index on older // servers) — fall through to recreate (if authorised) rather than fail outright. @@ -1071,17 +1286,19 @@ private void reconcileIndexConflict(PersistentEntity entity, collection.dropIndex(existingName); collection.createIndex(keys, desired); LOG.info("Recreated index [{}] on entity [{}] {}", existingName, entity.getName(), descriptor); + return true; } catch (MongoCommandException recreateError) { LOG.error("Failed to recreate index [{}] on entity [{}] {}: {}", existingName, entity.getName(), descriptor, recreateError.getMessage(), recreateError); + return false; } - return; } LOG.error( "Index conflict for entity [{}] {}: an index [{}] already exists on the same keys with different options. " + "Declare indexAttributes:[recreateOnConflict:true] to drop and recreate it. Original error: {}", entity.getName(), descriptor, existingName, original.getMessage()); + return false; } /** @@ -1094,9 +1311,9 @@ private void reconcileIndexConflict(PersistentEntity entity, * existing text index is unambiguously the one a newly-declared text index conflicts with — * match it regardless of its key shape or name so {@code recreateOnConflict} can absorb it.

*/ - private static Document findIndexByKeyPattern(com.mongodb.client.MongoCollection collection, Document keys) { + private static Document findIndexByKeyPattern(Iterable indexes, Document keys) { boolean desiredIsText = isTextIndex(keys); - for (Document idx : collection.listIndexes()) { + for (Document idx : indexes) { Object key = idx.get("key"); if (!(key instanceof Document)) { continue; @@ -1232,6 +1449,11 @@ private boolean ownsClient() { @PreDestroy public void close() { MongoClient current = this.mongo; + if (indexBuildExecutor != null) { + // Interrupt rather than wait: an index build can run for minutes and shutdown must not wait + // for it. The server carries on building what it was asked for. + indexBuildExecutor.shutdownNow(); + } try { super.destroy(); } catch (Exception e) { @@ -1260,6 +1482,28 @@ public void close() { } } + /** + * Names the background index build thread after the connection it serves, so that a log line or a + * thread dump says which datastore is building indexes. The thread is a daemon: an index build in + * flight must not hold the JVM open, and abandoning the wait does not abandon the build — the server + * finishes an index it has been asked for whether or not a client is still listening. + */ + private static final class IndexBuildThreadFactory implements ThreadFactory { + + private final String connectionName; + + private IndexBuildThreadFactory(String connectionName) { + this.connectionName = connectionName; + } + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "gorm-mongo-index-build-" + connectionName); + thread.setDaemon(true); + return thread; + } + } + /** * Creates the connection sources for a {@link MongoClient}. * @@ -1273,7 +1517,11 @@ public void close() { * @return The {@link ConnectionSources} */ protected static ConnectionSources createDefaultConnectionSources(MongoClient mongoClient, PropertyResolver configuration, MongoMappingContext mappingContext, boolean closeable) { - MongoConnectionSourceSettings settings = new MongoConnectionSourceSettings(); + // Bound from the configuration rather than left at the defaults: the client is supplied here, but + // the settings that describe how the datastore behaves (stateless, transactional, buildIndexes, + // engine, flush mode) still come from grails.mongodb, exactly as they do when GORM creates the + // client itself. The connection details in them are unused - this client is already connected. + MongoConnectionSourceSettings settings = new MongoConnectionSourceSettingsBuilder(configuration).build(); settings.setDatabaseName(mappingContext.getDefaultDatabaseName()); ConnectionSource defaultConnectionSource = new DefaultConnectionSource<>(ConnectionSource.DEFAULT, mongoClient, settings, closeable); return new InMemoryConnectionSources<>(defaultConnectionSource, new MongoConnectionSourceFactory(), configuration); diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/config/MongoSettings.groovy b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/config/MongoSettings.groovy index 4a2bbf15fbd..3da71ac088f 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/config/MongoSettings.groovy +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/config/MongoSettings.groovy @@ -95,6 +95,29 @@ interface MongoSettings extends Settings { String SETTING_ENGINE = 'grails.mongodb.engine' + /** + * Whether GORM creates and reconciles the indexes declared in domain class mapping blocks + * when the datastore starts. Defaults to {@code true}. + * + *

Set to {@code false} to leave the indexes on the server exactly as they are, which is + * useful when deploying against live data where index changes are applied separately by a + * DBA or a migration step rather than by the application on startup. + * + * @since 8.0 + */ + String SETTING_BUILD_INDEXES = 'grails.mongodb.buildIndexes' + + /** + * Whether the startup index build runs on a background thread rather than blocking the thread + * that creates the datastore. Defaults to {@code false}, which is the historical behavior: + * startup waits for MongoDB to finish building every declared index. + * + *

Has no effect when {@link #SETTING_BUILD_INDEXES} is {@code false}. + * + * @since 8.0 + */ + String SETTING_BUILD_INDEXES_ASYNC = 'grails.mongodb.buildIndexesAsync' + /** * Global default storage type for {@code String id} fields when no per-domain * {@code id storedAs: ...} mapping is declared. Accepted values are the names (or hex diff --git a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy index 09f6045427e..ab6957aeaa6 100644 --- a/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy +++ b/grails-data-mongodb/core/src/main/groovy/org/grails/datastore/mapping/mongo/connections/AbstractMongoConnectionSourceSettings.groovy @@ -97,6 +97,28 @@ abstract class AbstractMongoConnectionSourceSettings extends ConnectionSourceSet */ boolean decimalType = true + /** + * Whether GORM creates and reconciles the indexes declared in domain class mapping blocks + * when the datastore starts. When {@code false} the indexes on the server are left exactly + * as they are and no {@code createIndex} or {@code collMod} command is issued; queries are + * unaffected and continue to use whatever indexes already exist. Bound from + * {@code grails.mongodb.buildIndexes}. + * + * @since 8.0 + */ + boolean buildIndexes = true + + /** + * Whether the startup index build runs on a background thread instead of blocking the thread that + * creates the datastore. MongoDB answers a {@code createIndex} command only once the index has been + * built, so with the default {@code false} an application waits at startup for every declared index. + * Ignored when {@link #buildIndexes} is {@code false}. Bound from + * {@code grails.mongodb.buildIndexesAsync}. + * + * @since 8.0 + */ + boolean buildIndexesAsync = false + /** * Nested settings for domains with {@code String id}. Holds the global default * {@code defaultStoredAs} switch plus any future string-id configuration. diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy new file mode 100644 index 00000000000..32f83f75bdf --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesAsyncSpec.groovy @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import java.util.concurrent.ConcurrentLinkedQueue + +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import com.mongodb.event.CommandListener +import com.mongodb.event.CommandStartedEvent +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * MongoDB answers a {@code createIndexes} command only once the index has been built, so by default + * whoever creates the datastore - in an application, the startup thread - waits for every declared index. + * This specification pins both halves of that: the default build runs on the calling thread, and with + * {@code grails.mongodb.buildIndexesAsync = true} it runs on a background thread instead. + * + *

Which thread issued the command is observed through a driver {@link CommandListener}, which the + * synchronous driver invokes on the thread making the call. + */ +class BuildIndexesAsyncSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore blockingDatastore + + @Shared + @AutoCleanup + MongoDatastore asyncDatastore + + @Shared + MongoClient blockingClient + + @Shared + MongoClient asyncClient + + @Shared + CreateIndexThreadRecorder blockingRecorder = new CreateIndexThreadRecorder() + + @Shared + CreateIndexThreadRecorder asyncRecorder = new CreateIndexThreadRecorder() + + @Shared + String creatingThread + + @Override + boolean shouldInitializeDatastore() { + false + } + + private MongoClient clientFor(String database, CommandListener listener) { + MongoClients.create(MongoClientSettings.builder() + .applyConnectionString(new ConnectionString(dbContainer.getReplicaSetUrl(database))) + .addCommandListener(listener) + .build()) + } + + void setupSpec() { + creatingThread = Thread.currentThread().name + + blockingClient = clientFor('blockingIndexDb', blockingRecorder) + blockingDatastore = new MongoDatastore(blockingClient, + DatastoreUtils.createPropertyResolver(['grails.mongodb.databaseName': 'blockingIndexDb']), + BlockingIndexThing) + + asyncClient = clientFor('asyncIndexDb', asyncRecorder) + asyncDatastore = new MongoDatastore(asyncClient, + DatastoreUtils.createPropertyResolver([ + 'grails.mongodb.databaseName' : 'asyncIndexDb', + (MongoSettings.SETTING_BUILD_INDEXES_ASYNC): true + ]), + AsyncIndexThing) + } + + void cleanupSpec() { + blockingClient?.close() + asyncClient?.close() + } + + void "test the index build blocks the thread creating the datastore by default"() { + expect: "the setting is off" + !blockingDatastore.isBuildIndexesAsync() + + and: "the index already exists by the time the constructor has returned" + [name: 1] in BlockingIndexThing.collection.listIndexes()*.key + + and: "it was built by the thread that created the datastore, which therefore waited for it" + blockingRecorder.threads == [creatingThread] + } + + void "test the index build runs on a background thread when enabled"() { + given: + def conditions = new PollingConditions(timeout: 30) + + expect: "the setting is on" + asyncDatastore.isBuildIndexesAsync() + + when: "the background build has had a chance to run" + conditions.eventually { + assert [name: 1] in AsyncIndexThing.collection.listIndexes()*.key + } + + then: "the command was issued by the datastore's own index build thread" + asyncRecorder.threads.size() == 1 + asyncRecorder.threads.first().startsWith('gorm-mongo-index-build') + + and: "not by the thread that created the datastore, which did not wait for it" + asyncRecorder.threads.first() != creatingThread + } +} + +/** + * Records the thread each {@code createIndexes} command was issued from. + */ +class CreateIndexThreadRecorder implements CommandListener { + + private final Queue recorded = new ConcurrentLinkedQueue<>() + + @Override + void commandStarted(CommandStartedEvent event) { + if (event.commandName == 'createIndexes') { + recorded.add(Thread.currentThread().name) + } + } + + List getThreads() { + recorded.toList().unique() + } +} + +@Entity +class BlockingIndexThing { + String name + + static mapping = { + version false + collection 'blockingIndexThing' + name index: true + } +} + +@Entity +class AsyncIndexThing { + String name + + static mapping = { + version false + collection 'asyncIndexThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy new file mode 100644 index 00000000000..188a4affa74 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesDisabledSpec.groovy @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * Verifies that with {@code grails.mongodb.buildIndexes = false} GORM issues no index commands: the + * indexes declared in the mapping block are neither created nor reconciled, so whatever indexes are + * already on the server are left untouched. Persistence and querying are unaffected. + * + * @see BuildIndexesEnabledByDefaultSpec for the default behavior with the same mapping declarations + */ +class BuildIndexesDisabledSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + Map config = [ + 'grails.mongodb.url' : dbContainer.getReplicaSetUrl('myDb'), + (MongoSettings.SETTING_BUILD_INDEXES) : false + ] + datastore = new MongoDatastore(config, SkippedIndexThing) + } + + private static List declaredIndexKeys() { + SkippedIndexThing.withNewSession { + SkippedIndexThing.collection.listIndexes()*.key + } + } + + void "test the datastore reports index building disabled"() { + expect: + !datastore.isBuildIndexes() + } + + void "test no declared index is created on startup"() { + when: "a document is written so the collection certainly exists" + SkippedIndexThing.withNewSession { + new SkippedIndexThing(name: 'Fred', age: 42).save(flush: true) + } + + then: "only the implicit _id index is present - neither the property index nor the compound index was created" + declaredIndexKeys() == [[_id: 1]] + } + + void "test an explicit index build is a no-op while disabled"() { + when: "index building is requested directly" + datastore.buildIndex() + + then: "no index was created" + declaredIndexKeys() == [[_id: 1]] + } + + void "test registering an entity after startup does not create its indexes"() { + given: + def entity = datastore.mappingContext.getPersistentEntity(SkippedIndexThing.name) + + when: "the entity is re-registered, the path that indexes a domain class added after startup" + datastore.persistentEntityAdded(entity) + + then: "no index was created" + declaredIndexKeys() == [[_id: 1]] + } + + void "test reads and writes are unaffected when index building is disabled"() { + when: + SkippedIndexThing.withNewSession { + new SkippedIndexThing(name: 'Wilma', age: 41).save(flush: true) + } + + then: "the unindexed property is still queryable" + SkippedIndexThing.withNewSession { SkippedIndexThing.findByName('Wilma') }?.age == 41 + } +} + +@Entity +class SkippedIndexThing { + String name + Integer age + + static mapping = { + version false + collection 'skippedIndexThing' + name index: true + compoundIndex name: 1, age: -1 + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy new file mode 100644 index 00000000000..a2ac56c338d --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesEnabledByDefaultSpec.groovy @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * The control for {@link BuildIndexesDisabledSpec}: with {@code grails.mongodb.buildIndexes} left at its + * default the very same mapping declarations do produce indexes on startup, so the absence of indexes in + * that specification is attributable to the setting and not to the mapping. + */ +class BuildIndexesEnabledByDefaultSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + // No grails.mongodb.buildIndexes => default true + datastore = new MongoDatastore(['grails.mongodb.url': dbContainer.getReplicaSetUrl('myDb')] as Map, BuiltIndexThing) + } + + void "test index building is enabled by default"() { + expect: + datastore.isBuildIndexes() + } + + void "test the declared indexes are created on startup"() { + when: + List indexKeys = BuiltIndexThing.withNewSession { + BuiltIndexThing.collection.listIndexes()*.key + } + + then: "the property index and the compound index are both present alongside the implicit _id index" + [name: 1] in indexKeys + [name: 1, age: -1] in indexKeys + } +} + +@Entity +class BuiltIndexThing { + String name + Integer age + + static mapping = { + version false + collection 'builtIndexThing' + name index: true + compoundIndex name: 1, age: -1 + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy new file mode 100644 index 00000000000..df881733c0a --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/BuildIndexesSummaryLogSpec.groovy @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import grails.gorm.annotation.Entity +import org.slf4j.LoggerFactory +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.AbstractDatastore +import org.grails.datastore.mapping.mongo.MongoDatastore + +/** + * An index build that succeeds says so once, at the end: what it created, what was already there, how many + * domain classes it covered, and how long the caller spent waiting. That summary is the only signal a + * background build has finished at all, and the created/already-present split is what makes the elapsed + * time interpretable — a build that created nothing had nothing to wait for. + */ +class BuildIndexesSummaryLogSpec extends AutoStartedMongoSpec { + + static final String DATABASE = 'summaryLogDb' + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + Logger datastoreLogger + + @Shared + ListAppender logged = new ListAppender<>() + + @Shared + Level previousLevel + + @Shared + List startupMessages + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + // The datastore logs through the logger its base class declares + datastoreLogger = LoggerFactory.getLogger(AbstractDatastore) as Logger + previousLevel = datastoreLogger.level + datastoreLogger.level = Level.INFO + logged.start() + datastoreLogger.addAppender(logged) + + datastore = new MongoDatastore( + ['grails.mongodb.url': dbContainer.getReplicaSetUrl(DATABASE)] as Map, + SummaryLoggedThing, OtherSummaryLoggedThing) + + // Snapshotted so that a feature triggering another build cannot change what the startup build said + startupMessages = messagesForThisDatabase() + } + + void cleanupSpec() { + datastoreLogger?.detachAppender(logged) + datastoreLogger?.level = previousLevel + } + + /** + * Other specifications create datastores of their own in this JVM, so the summaries are picked out by + * the database this specification uses rather than by being the only messages logged. + */ + private List messagesForThisDatabase() { + logged.list.collect { it.formattedMessage }.findAll { it.contains("database [$DATABASE]") } + } + + void "test a successful index build logs one summary of what it applied and what it cost"() { + given: + String summary = startupMessages.first() + + expect: "exactly one summary for the build, not one line per index" + startupMessages.size() == 1 + + and: "it counted every declared index as created: two on one domain class, one on the other" + summary.contains('3 created, 0 already present') + + and: "and the domain classes they came from" + summary.contains('from 2 domain class(es)') + + and: "and how long the caller waited" + summary ==~ /Index build for database \[$DATABASE] finished in \d+ms: .*/ + } + + void "test a repeated build reports the indexes as already present rather than created"() { + given: "the summaries logged so far" + int before = messagesForThisDatabase().size() + + when: "the same declarations are applied again, as they would be on the next restart" + datastore.buildIndex() + + then: "the build reports that it created nothing, which is why it cost next to nothing" + List since = messagesForThisDatabase().drop(before) + since.size() == 1 + since.first().contains('0 created, 3 already present') + } +} + +@Entity +class SummaryLoggedThing { + String name + Integer age + + static mapping = { + version false + collection 'summaryLoggedThing' + name index: true + compoundIndex name: 1, age: -1 + } +} + +@Entity +class OtherSummaryLoggedThing { + String title + + static mapping = { + version false + collection 'otherSummaryLoggedThing' + title index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesPerConnectionSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesPerConnectionSpec.groovy new file mode 100644 index 00000000000..68845a635f1 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/BuildIndexesPerConnectionSpec.groovy @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo.connections + +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.connections.ConnectionSource +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * Verifies that {@code buildIndexes} is resolved per connection: a connection inherits the top level + * setting unless it declares its own, so index building can be switched off globally and left on for an + * individual connection (or the other way round). + */ +class BuildIndexesPerConnectionSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + Map config = [ + 'grails.mongodb.url' : "mongodb://${mongoHost}:${mongoPort}/skippedDb" as String, + (MongoSettings.SETTING_BUILD_INDEXES): false, + 'grails.mongodb.connections' : [ + 'indexed': [ + 'url' : "mongodb://${mongoHost}:${mongoPort}/indexedDb" as String, + 'buildIndexes': true + ], + 'inherits': [ + 'url': "mongodb://${mongoHost}:${mongoPort}/inheritsDb" as String + ] + ] + ] + datastore = new MongoDatastore(config, PerConnectionThing) + } + + void "test a connection can override the global setting"() { + expect: "the default connection is disabled by the top level setting" + !datastore.isBuildIndexes() + + and: "the connection that declares its own setting builds indexes" + datastore.getDatastoreForConnection('indexed').isBuildIndexes() + + and: "a connection that declares nothing inherits the top level setting" + !datastore.getDatastoreForConnection('inherits').isBuildIndexes() + } + + void "test only the connection with index building enabled has the declared index"() { + when: "a document is written through each connection so every collection exists" + PerConnectionThing.withNewSession { + new PerConnectionThing(name: 'Fred').save(flush: true) + } + PerConnectionThing.indexed.withNewSession { + new PerConnectionThing(name: 'Fred').save(flush: true) + } + + then: "the disabled connection has only the implicit _id index" + PerConnectionThing.collection.listIndexes()*.key == [[_id: 1]] + + and: "the enabled connection has the index declared in the mapping" + [name: 1] in PerConnectionThing.indexed.collection.listIndexes()*.key + } +} + +@Entity +class PerConnectionThing { + String name + + static mapping = { + version false + collection 'perConnectionThing' + connection ConnectionSource.ALL + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy new file mode 100644 index 00000000000..74ec48eaa48 --- /dev/null +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/gorm/mongo/connections/SuppliedMongoClientSettingsSpec.groovy @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.grails.datastore.gorm.mongo.connections + +import com.mongodb.client.MongoClient +import com.mongodb.client.MongoClients +import grails.gorm.annotation.Entity +import spock.lang.AutoCleanup +import spock.lang.Shared + +import org.apache.grails.testing.mongo.AutoStartedMongoSpec +import org.grails.datastore.mapping.core.DatastoreUtils +import org.grails.datastore.mapping.mongo.MongoDatastore +import org.grails.datastore.mapping.mongo.config.MongoSettings + +/** + * An application that hands GORM an existing {@code MongoClient} - which is what happens whenever a + * {@code MongoClient} bean is already present, as with Spring Boot's MongoDB auto-configuration - must + * still have its {@code grails.mongodb} settings applied. Only the connection details are taken from the + * supplied client; everything describing how the datastore behaves still comes from the configuration. + */ +class SuppliedMongoClientSettingsSpec extends AutoStartedMongoSpec { + + @Shared + @AutoCleanup + MongoDatastore datastore + + @Shared + MongoClient mongoClient + + @Override + boolean shouldInitializeDatastore() { + false + } + + void setupSpec() { + mongoClient = MongoClients.create(dbContainer.getReplicaSetUrl('suppliedClientDb')) + Map config = [ + 'grails.mongodb.databaseName' : 'suppliedClientDb', + (MongoSettings.SETTING_BUILD_INDEXES): false, + 'grails.mongodb.transactional' : true + ] + datastore = new MongoDatastore(mongoClient, DatastoreUtils.createPropertyResolver(config), SuppliedClientThing) + } + + void cleanupSpec() { + mongoClient?.close() + } + + void "test the configured settings are applied to a datastore built on a supplied client"() { + expect: "the index setting is taken from the configuration rather than left at its default" + !datastore.isBuildIndexes() + + and: "so is any other datastore setting" + datastore.isTransactionsEnabled() + } + + void "test the configured settings take effect and not merely report"() { + when: "a document is written so the collection certainly exists" + SuppliedClientThing.withNewSession { + new SuppliedClientThing(name: 'Fred').save(flush: true) + } + + then: "the index declared in the mapping was not created, as configured" + SuppliedClientThing.collection.listIndexes()*.key == [[_id: 1]] + } +} + +@Entity +class SuppliedClientThing { + String name + + static mapping = { + version false + collection 'suppliedClientThing' + name index: true + } +} diff --git a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy index c28ccafd37e..0bfd431e1a4 100644 --- a/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy +++ b/grails-data-mongodb/core/src/test/groovy/org/grails/datastore/mapping/mongo/config/MongoConnectionSourceSettingsSpec.groovy @@ -49,6 +49,36 @@ class MongoConnectionSourceSettingsSpec extends Specification { settings.options.build().readPreference == ReadPreference.secondary() } + void "test index building is enabled unless it is switched off in configuration"() { + when: "no buildIndexes setting is supplied" + def settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([:])).build() + + then: "declared indexes are built on startup" + settings.buildIndexes + + when: "the setting is switched off" + def resolver = DatastoreUtils.createPropertyResolver([(MongoSettings.SETTING_BUILD_INDEXES): 'false']) + settings = new MongoConnectionSourceSettingsBuilder(resolver).build() + + then: "index building is disabled" + !settings.buildIndexes + } + + void "test the index build is synchronous unless it is switched to asynchronous in configuration"() { + when: "no buildIndexesAsync setting is supplied" + def settings = new MongoConnectionSourceSettingsBuilder(DatastoreUtils.createPropertyResolver([:])).build() + + then: "the index build blocks the thread creating the datastore" + !settings.buildIndexesAsync + + when: "the setting is switched on" + def resolver = DatastoreUtils.createPropertyResolver([(MongoSettings.SETTING_BUILD_INDEXES_ASYNC): 'true']) + settings = new MongoConnectionSourceSettingsBuilder(resolver).build() + + then: "the index build is asynchronous" + settings.buildIndexesAsync + } + void "test mongo client settings builder with URL"() { when:"using a property resolver" Map myMap = ['grails.mongodb.url': 'mongodb://foo:bar@mycompany/mydb?maxPoolSize=5'] diff --git a/grails-data-mongodb/core/src/test/resources/logback-test.xml b/grails-data-mongodb/core/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..4d81bf0a99a --- /dev/null +++ b/grails-data-mongodb/core/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc index e3213027cb2..4db4c8ee2f5 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/gettingStarted/advancedConfig.adoc @@ -21,7 +21,20 @@ under the License. ==== Mongo Database Connection Configuration -As mentioned the GORM for MongoDB plugin will configure all the defaults for you, but if you wish to customize those defaults you can do so in the `grails-app/conf/application.groovy` file: +As mentioned the GORM for MongoDB plugin will configure all the defaults for you, but if you wish to customize those defaults you can do so in `grails-app/conf/application.yml`: + +[source,yaml] +---- +grails: + mongodb: + host: localhost + port: 27017 + username: blah + password: blah + databaseName: foo +---- + +or equivalently in `grails-app/conf/application.groovy`: [source,groovy] ---- @@ -36,6 +49,8 @@ grails { } ---- +NOTE: These settings are read by name, so write them exactly as they are documented. Unlike Spring Boot's own configuration properties they are not relaxed-bound, and a kebab-case spelling such as `database-name` is not recognised — it is ignored, leaving the default in place. + The `databaseName` setting configures the default database name. If not specified the `databaseName` will default to the name of your application. You can also customize the MongoDB connection settings using an `options` block: @@ -87,6 +102,8 @@ grails { username = ".." // the username to connect with password = ".." // the password to connect with stateless = false // whether to use stateless sessions by default + buildIndexes = true // whether to create the indexes declared in mapping blocks on startup + buildIndexesAsync = false // whether that index build runs on a background thread instead of blocking startup // Alternatively, using 'url' // url = "mongodb://localhost/mydb" @@ -125,6 +142,11 @@ The `*` method is used to indicate that the setting applies to all properties. You can also set a global default for the storage type of `String id` fields via `grails.mongodb.stringIds.defaultStoredAs` (values `string` or `objectid`). See <> for details. +==== Index Creation on Startup + + +GORM creates and reconciles the indexes declared in your mapping blocks each time the datastore starts, and startup waits for MongoDB to finish building them. Set `grails.mongodb.buildIndexes = false` to leave the indexes on the server untouched instead, or `grails.mongodb.buildIndexesAsync = true` to keep building them but on a background thread — see <> for details and when to use each. + ==== Multi-Document Transactions diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc index 243bb756785..eb5e122c9ad 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/introduction/releaseNotes.adoc @@ -25,6 +25,9 @@ Below are the details of the changes across releases: * TTL indexes via `indexAttributes: [expireAfterSeconds: N]` in the mapping DSL * Text and other special indexes via `indexAttributes: [type: 'text']` * In-place reconciliation of changed index options, with opt-in `indexAttributes: [recreateOnConflict: true]` +* Index creation on startup can be switched off with `grails.mongodb.buildIndexes = false`, or moved off the startup thread with `grails.mongodb.buildIndexesAsync = true` +* `grails.mongodb` settings are now applied when GORM is given an existing `MongoClient`, such as the one contributed by Spring Boot +* The index build reports what it created, what was already present and how long it took, in one summary line per build ==== 7.1 diff --git a/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc b/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc index 2c4bc61a020..1a797e017d4 100644 --- a/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc +++ b/grails-data-mongodb/docs/src/docs/asciidoc/querying/queryIndexes.adoc @@ -142,6 +142,93 @@ WARNING: Dropping and recreating an index rebuilds it from scratch, during which A change to a TTL index's `expireAfterSeconds` is handled automatically and does not require `recreateOnConflict`, because GORM updates the expiry in place rather than rebuilding the index. +==== Disabling Index Creation on Startup + + +By default GORM creates and reconciles every index declared in a mapping block when the datastore starts. Set `buildIndexes` to `false` in `grails-app/conf/application.yml` to switch that off: + +[source,yaml] +---- +grails: + mongodb: + buildIndexes: false +---- + +or, in `application.groovy`: + +[source,groovy] +---- +grails { + mongodb { + buildIndexes = false + } +} +---- + +With this setting no `createIndex` or `collMod` command is issued for any domain class, and the indexes already present on the server are left exactly as they are. Queries are unaffected and continue to use whichever indexes exist. This is useful when deploying against live data whose indexes are managed separately — by a DBA or a migration step — so that a deployment does not build an index against a large production collection, and so an application running against an older index set does not have those indexes reconciled underneath it. + +It also suppresses index creation for domain classes registered after startup. Declared at the top level the setting applies to every connection, and each connection can override it: + +[source,yaml] +---- +grails: + mongodb: + buildIndexes: false + connections: + reporting: + url: mongodb://localhost/reporting + buildIndexes: true +---- + +NOTE: Because nothing is created, a collection that has never been initialised with `buildIndexes` enabled will have no declared indexes at all. Turn the setting off only where the indexes are already in place or are applied by other means. The setting governs only the indexes GORM derives from the mapping blocks; an explicit `createIndex` call made by application code against a collection is unaffected. + +==== Building Indexes in the Background + + +MongoDB answers a `createIndex` command only once the index has been built, so by default the thread that creates the datastore — in an application, the startup thread — waits for every declared index before the application finishes starting. On an empty collection that is instant; on a large existing collection an index build can take minutes, and a deployment waits for all of them in turn. + +Set `buildIndexesAsync` to have the startup index build run on a background thread instead: + +[source,yaml] +---- +grails: + mongodb: + buildIndexesAsync: true +---- + +Startup then continues without waiting. The indexes are still built one at a time, on a single daemon thread per connection named `gorm-mongo-index-build-`, so enabling this does not launch several index builds against the server at once. + +Two consequences are worth planning for: + +* A query issued before its index has been built is served without it — correctly, but with the performance of an unindexed query. The same applies to a `unique` index: it constrains nothing until the build finishes. +* Because startup no longer waits, a failure to build an index can no longer fail startup. It is logged at error level, and the application runs without that index. With the default synchronous build the exception propagates and the application does not start. + +The setting is ignored when `buildIndexes` is `false`, and it applies only to the index build performed when the datastore starts — a domain class registered after startup is indexed on the thread registering it. + +NOTE: If the application shuts down while a background build is still running, GORM stops waiting for it, but the server carries on building the index it was asked for. + +==== What the Index Build Reports + + +An index build that finishes without error logs one summary line at `INFO`: + +---- +Index build for database [myDb] finished in 412ms: 2 created, 5 already present, from 3 domain class(es) +---- + +The elapsed time is what the caller actually spent waiting — startup with the default settings, or the background thread when `buildIndexesAsync` is enabled, where this line is also the only signal that the build has finished. + +The split between created and already present is what makes that time interpretable. MongoDB answers a `createIndex` for an index it already has immediately and without building anything, so a restart that changed no mappings reports everything as already present and costs milliseconds; a line reporting indexes created is the one that accounts for a slow start. If any declaration failed, the summary is logged at `WARN` instead and reports how many; the failures themselves are logged individually as they happen. + +To find which index is the slow one, enable `DEBUG` logging for `org.grails.datastore.mapping.core`, which adds a line per index with its own elapsed time: + +[source,yaml] +---- +logging: + level: + org.grails.datastore.mapping.core: DEBUG +---- + ==== Indexing using the 'index' method