Merge pull request #327 from entando/ENG-4265-liquibase-force-release…

…-lock

ENG-4265 Released Liquibase lock at application startup

src/main/java/org/entando/entando/aps/system/init/DatabaseManager.java

@@ -27,6 +27,8 @@
 import java.sql.DatabaseMetaData;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -45,6 +47,7 @@
 import liquibase.database.jvm.JdbcConnection;
 import liquibase.exception.DatabaseException;
 import liquibase.exception.LiquibaseException;
+import liquibase.lockservice.DatabaseChangeLogLock;
 import liquibase.resource.ClassLoaderResourceAccessor;
 import org.apache.commons.beanutils.BeanComparator;
 import org.apache.commons.io.output.StringBuilderWriter;
@@ -87,6 +90,7 @@ public class DatabaseManager extends AbstractInitializerManager
     private DatabaseDumper databaseDumper;
     private DatabaseRestorer databaseRestorer;
     private List<DataSource> defaultDataSources;
+    private int lockFallbackMinutes;
 
     private ServletContext servletContext;
 
@@ -261,6 +265,7 @@ private List<ChangeSetStatus> executeLiquibaseUpdate(Date timestamp, String comp
             JdbcConnection liquibaseConnection = new JdbcConnection(connection);
             Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(liquibaseConnection);
             liquibase = new Liquibase(changeLogFile, new ClassLoaderResourceAccessor(), database); // NOSONAR
+            this.releaseLockIfNeeded(liquibase);
             Contexts contexts = getContexts(status);
             if (DatabaseMigrationStrategy.AUTO.equals(migrationStrategy)) {
                 liquibase.update(contexts, new LabelExpression());
@@ -298,6 +303,25 @@ private List<ChangeSetStatus> executeLiquibaseUpdate(Date timestamp, String comp
         return changeSetToExecute;
     }
 
+    /**
+     * Checks if the application is stuck waiting for changelog lock release
+     * for too much time and in that case it forces the release of the lock.
+     * @param liquibase
+     */
+    private void releaseLockIfNeeded(Liquibase liquibase) throws LiquibaseException {
+        Instant releaseLockLimit = Instant.now().minus(lockFallbackMinutes, ChronoUnit.MINUTES);
+        for (DatabaseChangeLogLock lock : liquibase.listLocks()) {
+            if (lock.getLockGranted().toInstant().isBefore(releaseLockLimit)) {
+                logger.warn("A Liquibase lock older than {} minutes has been detected. Locks are being forcedly released.", lockFallbackMinutes);
+                liquibase.forceReleaseLocks();
+                break;
+            } else {
+                logger.warn("A Liquibase lock has been detected but it has not been released since it was "
+                        + "created less than {} minutes ago, that is the configured waiting time", lockFallbackMinutes);
+            }
+        }
+    }
+
     private Contexts getContexts(Status status) {
         String context = null;
         if (status.equals(Status.RESTORE)) {
@@ -606,4 +630,9 @@ public void setServletContext(ServletContext servletContext) {
     public void setDefaultDataSources(List<DataSource> defaultDataSources) {
         this.defaultDataSources = defaultDataSources;
     }
+
+    @Override
+    public void setLockFallbackMinutes(int lockFallbackMinutes) {
+        this.lockFallbackMinutes = lockFallbackMinutes;
+    }
 }

src/main/java/org/entando/entando/aps/system/init/IDatabaseManager.java

@@ -41,6 +41,8 @@ public interface IDatabaseManager {
 	public List<DataSourceDumpReport> getBackupReports() throws EntException;
 
 	public DatabaseType getDatabaseType(DataSource dataSource) throws EntException;
+
+	public void setLockFallbackMinutes(int lockFallbackMinutes);
 
 	public enum DatabaseType {DERBY, POSTGRESQL, MYSQL, ORACLE, SQLSERVER, UNKNOWN}
 

src/main/resources/spring/baseSystemConfig.xml

@@ -130,6 +130,9 @@
                 <ref bean="servDataSource"/>
             </list>
         </property>
+        <property name="lockFallbackMinutes">
+            <value>${entando.liquibase.lock.fallback.minutes:10}</value>
+        </property>
     </bean>
 
     <bean id="SelfRestCaller" class="org.entando.entando.aps.system.init.SelfRestCaller" />

src/test/java/org/entando/entando/aps/system/init/DatabaseManagerTest.java

@@ -18,9 +18,12 @@
 import java.io.InputStream;
 import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,6 +33,7 @@
 import liquibase.changelog.ChangeSetStatus;
 import liquibase.database.DatabaseFactory;
 import liquibase.exception.DatabaseException;
+import liquibase.lockservice.DatabaseChangeLogLock;
 import org.entando.entando.aps.system.init.AbstractInitializerManager.Environment;
 import org.entando.entando.aps.system.init.IInitializerManager.DatabaseMigrationStrategy;
 import org.entando.entando.aps.system.init.exception.DatabaseMigrationException;
@@ -57,6 +61,7 @@
 import org.mockito.Mockito;
 import org.mockito.Spy;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.verification.VerificationMode;
 import org.springframework.beans.factory.ListableBeanFactory;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
@@ -197,6 +202,7 @@ void testGenerateSqlStrategy() throws Exception {
         try (MockedConstruction<Liquibase> construction = Mockito.mockConstruction(Liquibase.class, (liquibase, context) -> {
             Mockito.when(liquibase.getChangeSetStatuses(ArgumentMatchers.any(), ArgumentMatchers.any()))
                     .thenReturn(Arrays.asList(changeSetStatus));
+            Mockito.when(liquibase.listLocks()).thenReturn(new DatabaseChangeLogLock[]{});
         }); MockedStatic<DatabaseFactory> dbFactory = Mockito.mockStatic(DatabaseFactory.class)) {
             dbFactory.when(DatabaseFactory::getInstance).thenReturn(Mockito.mock(DatabaseFactory.class));
 
@@ -207,6 +213,38 @@ void testGenerateSqlStrategy() throws Exception {
         }
     }
 
+    @Test
+    void testForceReleaseLockIsNotNeeded() throws Exception {
+        testForceReleaseLock(2, Mockito.never());
+    }
+
+    @Test
+    void testForceReleaseLockIsNeeded() throws Exception {
+        testForceReleaseLock(30, Mockito.times(1));
+    }
+
+    private void testForceReleaseLock(int minutes, VerificationMode mode) throws Exception {
+        databaseManager.setLockFallbackMinutes(10);
+
+        DatabaseChangeLogLock lock = Mockito.mock(DatabaseChangeLogLock.class);
+        Date lockGranted = Date.from(Instant.now().minus(minutes, ChronoUnit.MINUTES));
+        Mockito.when(lock.getLockGranted()).thenReturn(lockGranted);
+
+        ChangeSetStatus changeSetStatus = Mockito.mock(ChangeSetStatus.class);
+        getMockedBeanFactory();
+
+        try (MockedConstruction<Liquibase> construction = Mockito.mockConstruction(Liquibase.class,
+                (liquibase, context) -> {
+                    Mockito.when(liquibase.getChangeSetStatuses(ArgumentMatchers.any(), ArgumentMatchers.any()))
+                            .thenReturn(Arrays.asList(changeSetStatus));
+                    Mockito.when(liquibase.listLocks()).thenReturn(new DatabaseChangeLogLock[]{lock});
+                }); MockedStatic<DatabaseFactory> dbFactory = Mockito.mockStatic(DatabaseFactory.class)) {
+            dbFactory.when(DatabaseFactory::getInstance).thenReturn(Mockito.mock(DatabaseFactory.class));
+            databaseManager.installDatabase(getMockedReport(), DatabaseMigrationStrategy.AUTO);
+            Mockito.verify(construction.constructed().get(0), mode).forceReleaseLocks();
+        }
+    }
+
     @Test
     void testDerbyErrorOnLiquibaseClose() throws Throwable {
         testLiquibaseCloseException(new DatabaseException("Error closing derby cleanly"));