Merge remote-tracking branch 'upstream/develop' into issues/merge-ups…

…tream

docs/db/changelog/changesets/enrollment-server-onboarding/1.5.x/20230530-add-column-total-attempts.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.9.xsd">
+
+    <changeSet id="1" logicalFilePath="enrollment-server/1.5.x/20230530-add-column-total-attempts.xml" author="Jan Dusil">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <columnExists tableName="es_onboarding_otp" columnName="total_attempts"/>
+            </not>
+        </preConditions>
+        <comment>Add total_attempts column</comment>
+        <addColumn tableName="es_onboarding_otp">
+            <column name="total_attempts" type="integer" defaultValueNumeric="0" />
+        </addColumn>
+    </changeSet>
+</databaseChangeLog>

docs/db/changelog/changesets/enrollment-server-onboarding/1.5.x/db.changelog-version.xml

@@ -4,5 +4,6 @@
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.9.xsd">
 
     <include file="20230315-add-column-fds-data.xml" relativeToChangelogFile="true" />
+    <include file="20230530-add-column-total-attempts.xml" relativeToChangelogFile="true" />
 
 </databaseChangeLog>

docs/onboarding/Database-Structure.md

@@ -26,11 +26,11 @@ Only one ShedLock table is required per PowerAuth stack in case the same schema
 
 ## Auditing
 
-The DDL files contain an `audit_log` table definition. The table differs slightly per database. 
+The DDL files contain an `audit_log` table definition. The table differs slightly per database.
 
 Only one `audit_log` table is required per PowerAuth stack in case the same schema is used for all deployed applications.
 
-For more information about auditing library, see the [Wultra auditing library documentation](https://github.com/wultra/lime-java-core#wultra-auditing-library). 
+For more information about auditing library, see the [Wultra auditing library documentation](https://github.com/wultra/lime-java-core#wultra-auditing-library).
 
 ## Table Documentation
 
@@ -78,7 +78,8 @@ Stores onboarding OTP codes used during activation and user verification.
 | `type` | `VARCHAR(32)` | `NOT NULL` | OTP code type (`ACTIVATION`, `USER_VERIFICATION`). |
 | `error_detail` | `VARCHAR(256)` |  | Detail of error (e.g. information about timeout or exceeded number of failed attempts). |
 | `error_origin` | `VARCHAR(256)` |  | Origin of the error (`DOCUMENT_VERIFICATION`, `PRESENCE_CHECK`, `CLIENT_EVALUATION`, `OTP_VERIFICATION`, `PROCESS_LIMIT_CHECK`, `USER_REQUEST`). |
-| `failed_attempts` | `INTEGER` |  | Number of failed attempts for verification, |
+| `failed_attempts` | `INTEGER` |  | Number of failed attempts for verification. |
+| `total_attempts` | `INTEGER` |  | Number of total attempts for verification. |
 | `timestamp_created` | `TIMESTAMP` | `NOT NULL DEFAULT CURRENT_TIMESTAMP` | Timestamp when process was started. |
 | `timestamp_expiration` | `TIMESTAMP` | `NOT NULL` | Timestamp when the OTP expires. |
 | `timestamp_last_updated` | `TIMESTAMP` |  | Timestamp when record was last updated. |

docs/onboarding/PowerAuth-Enrollment-Onboarding-Server-1.5.0.md

@@ -54,6 +54,26 @@ ALTER TABLE es_onboarding_process
     ADD fds_data CLOB;
 ```
 
+### Total Attempts
+
+A new column `total_attempts` has been added to the table `es_onboarding_otp`.
+
+
+#### PostgreSQL
+
+```sql
+ALTER TABLE es_onboarding_process
+    ADD COLUMN TOTAL_ATTEMPTS INTEGER DEFAULT 0;
+```
+
+
+#### Oracle
+
+```sql
+ALTER TABLE es_onboarding_process
+    ADD total_attempts INTEGER DEFAULT 0;
+```
+
 
 ## Dependencies
 

docs/sql/oracle/onboarding/create-schema.sql

@@ -51,6 +51,7 @@ CREATE TABLE ES_ONBOARDING_OTP (
     ERROR_DETAIL VARCHAR2(256 CHAR),
     ERROR_ORIGIN VARCHAR2(256 CHAR),
     FAILED_ATTEMPTS INTEGER,
+    TOTAL_ATTEMPTS INTEGER DEFAULT 0,
     TIMESTAMP_CREATED TIMESTAMP(6) NOT NULL,
     TIMESTAMP_EXPIRATION TIMESTAMP(6) NOT NULL,
     TIMESTAMP_LAST_UPDATED TIMESTAMP(6),

docs/sql/postgresql/onboarding/create-schema.sql

@@ -55,6 +55,7 @@ CREATE TABLE es_onboarding_otp (
     error_detail VARCHAR(256),
     error_origin VARCHAR(256),
     failed_attempts INTEGER,
+    total_attempts INTEGER DEFAULT 0,
     timestamp_created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     timestamp_expiration TIMESTAMP NOT NULL,
     timestamp_last_updated TIMESTAMP,

enrollment-server-onboarding-common/src/main/java/com/wultra/app/onboardingserver/common/database/OnboardingProcessRepository.java

@@ -89,8 +89,7 @@ public interface OnboardingProcessRepository extends CrudRepository<OnboardingPr
      */
     @Lock(LockModeType.PESSIMISTIC_WRITE)
     @Query("SELECT p FROM OnboardingProcessEntity p WHERE p.status = :status " +
-            "AND p.activationId = :activationId " +
-            "ORDER BY p.timestampCreated DESC")
+            "AND p.activationId = :activationId")
     Optional<OnboardingProcessEntity> findByActivationIdAndStatusWithLock(String activationId, OnboardingStatus status);
 
     /**

enrollment-server-onboarding-common/src/main/java/com/wultra/app/onboardingserver/common/database/entity/OnboardingOtpEntity.java

@@ -92,6 +92,9 @@ public class OnboardingOtpEntity implements Serializable {
     @Column(name = "failed_attempts")
     private int failedAttempts;
 
+    @Column(name = "total_attempts")
+    private int totalAttempts;
+
     @Column(name = "timestamp_created", nullable = false)
     private Date timestampCreated;
 

enrollment-server-onboarding-common/src/main/java/com/wultra/app/onboardingserver/common/service/CommonOtpService.java

@@ -118,17 +118,20 @@ protected OtpVerifyResponse verifyOtpCode(String processId, OwnerId ownerId, Str
             otp.setErrorOrigin(ErrorOrigin.OTP_VERIFICATION);
             otp.setTimestampLastUpdated(now);
             otp.setTimestampFailed(now);
+            otp.setTotalAttempts(otp.getTotalAttempts() + 1);
             onboardingOtpRepository.save(otp);
             auditService.audit(otp, "OTP expired for user: {}", process.getUserId());
         } else if (otp.getOtpCode().equals(otpCode)) {
             verified = true;
             otp.setStatus(OtpStatus.VERIFIED);
             otp.setTimestampVerified(now);
             otp.setTimestampLastUpdated(now);
+            otp.setTotalAttempts(otp.getTotalAttempts() + 1);
             onboardingOtpRepository.save(otp);
             logger.info("OTP {} verified, {}", otpType, ownerId);
             auditService.audit(otp, "OTP {} verified for user: {}", otpType, process.getUserId());
         } else {
+            auditService.audit(otp, "OTP {} verification failed for user: {}", otpType, process.getUserId());
             handleFailedOtpVerification(process, ownerId, otp, otpType);
         }
 
@@ -168,6 +171,7 @@ private void handleFailedOtpVerification(OnboardingProcessEntity process, OwnerI
         int failedAttempts = fetchFailedAttemptsCount(process, otpType);
         final int maxFailedAttempts = commonOnboardingConfig.getOtpMaxFailedAttempts();
         otp.setFailedAttempts(otp.getFailedAttempts() + 1);
+        otp.setTotalAttempts(otp.getTotalAttempts() + 1);
         otp.setTimestampLastUpdated(ownerId.getTimestamp());
         otp = onboardingOtpRepository.save(otp);
         failedAttempts++;

enrollment-server-onboarding/src/main/java/com/wultra/app/onboardingserver/impl/service/OtpServiceImpl.java

@@ -172,6 +172,7 @@ private String generateOtpCode(OnboardingProcessEntity process, OtpType otpType)
         otp.setTimestampCreated(timestampCreated);
         otp.setTimestampExpiration(timestampExpiration);
         otp.setFailedAttempts(0);
+        otp.setTotalAttempts(0);
 
         if (otpType == OtpType.USER_VERIFICATION) {
             final String activationId = process.getActivationId();

enrollment-server-onboarding/src/main/java/com/wultra/app/onboardingserver/impl/service/document/DocumentProcessingService.java

@@ -249,7 +249,7 @@ public DocumentSubmitResult submitDocumentToProvider(OwnerId ownerId, DocumentVe
         try {
             docsSubmitResults = documentVerificationProvider.submitDocuments(ownerId, List.of(submittedDoc));
             final IdentityVerificationEntity identityVerification = docVerification.getIdentityVerification();
-            auditService.auditDocumentVerificationProvider(identityVerification, "Submit documents for user: {}", ownerId.getUserId());
+            auditService.auditDocumentVerificationProvider(identityVerification, "Submit documents for user: {}, document ID: {}", ownerId.getUserId(), docVerification.getId());
             docSubmitResult = docsSubmitResults.getResults().get(0);
         } catch (DocumentVerificationException | RemoteCommunicationException e) {
             logger.warn("Document verification ID: {}, failed: {}", docVerification.getId(), e.getMessage());

enrollment-server-onboarding/src/main/java/com/wultra/app/onboardingserver/presencecheck/iproov/config/IProovConfig.java

@@ -25,13 +25,15 @@
 import com.wultra.core.rest.client.base.RestClient;
 import com.wultra.core.rest.client.base.RestClientConfiguration;
 import com.wultra.core.rest.client.base.RestClientException;
+import io.netty.channel.ChannelOption;
 import lombok.extern.slf4j.Slf4j;
 import net.minidev.json.JSONObject;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.http.HttpHeaders;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
 import org.springframework.security.oauth2.client.AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager;
 import org.springframework.security.oauth2.client.ClientCredentialsReactiveOAuth2AuthorizedClientProvider;
 import org.springframework.security.oauth2.client.InMemoryReactiveOAuth2AuthorizedClientService;
@@ -49,9 +51,14 @@
 import org.springframework.web.reactive.function.client.ClientResponse;
 import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
 import org.springframework.web.reactive.function.client.WebClient;
+import org.springframework.web.reactive.function.client.WebClientResponseException;
 import org.springframework.web.util.UriComponentsBuilder;
+import reactor.netty.http.client.HttpClient;
+import reactor.netty.resources.ConnectionProvider;
 
+import java.time.Duration;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * iProov configuration.
@@ -101,9 +108,7 @@ public WebClient iproovManagemenentWebClient(final IProovConfigProps configProps
         final ServerOAuth2AuthorizedClientExchangeFilterFunction oAuth2ExchangeFilterFunction = new ServerOAuth2AuthorizedClientExchangeFilterFunction(authorizedClientManager);
         oAuth2ExchangeFilterFunction.setDefaultClientRegistrationId(OAUTH_REGISTRATION_ID);
 
-        return WebClient.builder()
-                .filter(oAuth2ExchangeFilterFunction)
-                .build();
+        return createWebClient(oAuth2ExchangeFilterFunction, configProps, "user management client");
     }
 
     private static AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager authorizedClientServiceReactiveOAuth2AuthorizedClientManager(final IProovConfigProps configProps) {
@@ -124,7 +129,7 @@ private static AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager auth
         final ReactiveOAuth2AuthorizedClientService clientService = new InMemoryReactiveOAuth2AuthorizedClientService(clientRegistrations);
 
         final ClientCredentialsReactiveOAuth2AuthorizedClientProvider authorizedClientProvider = new ClientCredentialsReactiveOAuth2AuthorizedClientProvider();
-        authorizedClientProvider.setAccessTokenResponseClient(accessTokenResponseClient());
+        authorizedClientProvider.setAccessTokenResponseClient(accessTokenResponseClient(configProps));
 
         final AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager authorizedClientManager =
                 new AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager(clientRegistrations, clientService);
@@ -133,28 +138,55 @@ private static AuthorizedClientServiceReactiveOAuth2AuthorizedClientManager auth
     }
 
     // TODO (racansky, 2023-06-05) remove when iProov fix API according the RFC
-    private static ReactiveOAuth2AccessTokenResponseClient<OAuth2ClientCredentialsGrantRequest> accessTokenResponseClient() {
+    private static ReactiveOAuth2AccessTokenResponseClient<OAuth2ClientCredentialsGrantRequest> accessTokenResponseClient(final IProovConfigProps configProps) {
         @SuppressWarnings("unchecked")
         final ExchangeFilterFunction tokenResponseFilter = ExchangeFilterFunction.ofResponseProcessor(response -> {
-            final ClientResponse.Builder builder = response.mutate();
-            return response.bodyToMono(Map.class).map(map -> {
-                if (map.containsKey(AuthTokenResponse.JSON_PROPERTY_SCOPE)) {
-                    logger.debug("Removing scope because does not comply with RFC and not needed anyway");
-                    map.remove(AuthTokenResponse.JSON_PROPERTY_SCOPE);
-                    return builder.body(JSONObject.toJSONString(map)).build();
-                } else {
-                    return builder.build();
-                }
-            });
+        final ClientResponse.Builder builder = response.mutate();
+        return response.bodyToMono(Map.class).map(map -> {
+                    logger.trace("Got access token");
+                    if (map.containsKey(AuthTokenResponse.JSON_PROPERTY_SCOPE)) {
+                        logger.debug("Removing scope because does not comply with RFC and not needed anyway");
+                        map.remove(AuthTokenResponse.JSON_PROPERTY_SCOPE);
+                        return builder.body(JSONObject.toJSONString(map)).build();
+                    } else {
+                        return builder.build();
+                    }
+                })
+                .doOnError(e -> {
+                    if (e instanceof final WebClientResponseException exception) {
+                        logger.error("Get access token - Error response body: {}", exception.getResponseBodyAsString());
+                    } else {
+                        logger.error("Get access token - Error", e);
+                    }
+                });
         });
 
-        final WebClient webClient = WebClient.builder()
-                .filter(tokenResponseFilter)
-                .build();
+        final WebClient webClient = createWebClient(tokenResponseFilter, configProps, "oAuth client");
 
         final AbstractWebClientReactiveOAuth2AccessTokenResponseClient<OAuth2ClientCredentialsGrantRequest> accessTokenResponseClient = new WebClientReactiveClientCredentialsTokenResponseClient();
         accessTokenResponseClient.setWebClient(webClient);
         return accessTokenResponseClient;
     }
 
+    private static WebClient createWebClient(final ExchangeFilterFunction filter, IProovConfigProps configProps, final String logContext) {
+        final RestClientConfiguration restClientConfig = configProps.getRestClientConfig();
+        final Integer connectionTimeout = restClientConfig.getConnectionTimeout();
+        final Duration responseTimeout = restClientConfig.getResponseTimeout();
+        final Duration maxIdleTime = Objects.requireNonNull(restClientConfig.getMaxIdleTime(), "maxIdleTime must be specified");
+        logger.info("Setting {} connectionTimeout: {}, responseTimeout: {}, maxIdleTime: {}", logContext, connectionTimeout, responseTimeout, maxIdleTime);
+
+        final ConnectionProvider connectionProvider = ConnectionProvider.builder("custom")
+                .maxIdleTime(maxIdleTime)
+                .build();
+        final HttpClient httpClient = HttpClient.create(connectionProvider)
+                .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, connectionTimeout)
+                .responseTimeout(responseTimeout);
+        final ReactorClientHttpConnector connector = new ReactorClientHttpConnector(httpClient);
+
+        return WebClient.builder()
+                .clientConnector(connector)
+                .filter(filter)
+                .build();
+    }
+
 }

enrollment-server-onboarding/src/main/java/com/wultra/app/onboardingserver/presencecheck/iproov/service/IProovRestApiService.java

@@ -229,6 +229,8 @@ private boolean doesUserExists(final String userId, final OwnerId id) {
                 .doOnError(e -> {
                     if (e instanceof final WebClientResponseException exception) {
                         logger.error("Get user - Error response body: {}, userId: {}, {}", exception.getResponseBodyAsString(), userId, id);
+                    } else {
+                        logger.error("Get user - Error, userId: {}, {}", userId, id, e);
                     }
                 })
                 .block(), false);
@@ -241,6 +243,8 @@ private void deleteUser(final String userId, final OwnerId id) {
                 .doOnError(e -> {
                     if (e instanceof final WebClientResponseException exception) {
                         logger.error("Delete user - Error response body: {}, userId: {}, {}", exception.getResponseBodyAsString(), userId, id);
+                    } else {
+                        logger.error("Get user - Error, userId: {}, {}", userId, id, e);
                     }
                 })
                 .block();

enrollment-server-onboarding/src/main/resources/application.properties

@@ -175,6 +175,7 @@ enrollment-server-onboarding.presence-check.iproov.restClientConfig.proxyUsernam
 enrollment-server-onboarding.presence-check.iproov.restClientConfig.proxyPassword=
 enrollment-server-onboarding.presence-check.iproov.restClientConfig.connectionTimeout=10000
 enrollment-server-onboarding.presence-check.iproov.restClientConfig.responseTimeout=60000
+enrollment-server-onboarding.presence-check.iproov.restClientConfig.maxIdleTime=200s
 
 spring.security.oauth2.client.provider.app.token-uri=http://localhost:6060/oauth/token
 

enrollment-server-onboarding/src/test/resources/com/wultra/app/onboardingserver/task/cleaning/CleaningServiceTest.testTerminateExpiredOtpCodes.sql

@@ -1,7 +1,7 @@
 INSERT INTO es_onboarding_process(id, identification_data, status, error_score, custom_data, timestamp_created) VALUES
     ('b4662611-df91-4053-bb3d-3970979baf5d', '{}', 'VERIFICATION_IN_PROGRESS', 0, '{}', now());
 
-INSERT INTO es_onboarding_otp(id, process_id, otp_code, failed_attempts, status, type, timestamp_created, timestamp_expiration) VALUES
-    ('f50b8c04-649d-43a7-8079-4dbf9b0bbc72', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 'ACTIVE', 'USER_VERIFICATION', now(), now()),
-    ('6560a85d-7d97-44c0-bd29-04c57051aa57', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 'ACTIVE', 'USER_VERIFICATION', now() - interval '301' second, now()), -- to be failed
-    ('e4974ef6-135a-4ae1-be91-a2c0f674c8fd', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 'VERIFIED', 'USER_VERIFICATION', now() - interval '301' second, now());
+INSERT INTO es_onboarding_otp(id, process_id, otp_code, failed_attempts, total_attempts, status, type, timestamp_created, timestamp_expiration) VALUES
+    ('f50b8c04-649d-43a7-8079-4dbf9b0bbc72', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 0, 'ACTIVE', 'USER_VERIFICATION', now(), now()),
+    ('6560a85d-7d97-44c0-bd29-04c57051aa57', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 0, 'ACTIVE', 'USER_VERIFICATION', now() - interval '301' second, now()), -- to be failed
+    ('e4974ef6-135a-4ae1-be91-a2c0f674c8fd', 'b4662611-df91-4053-bb3d-3970979baf5d', '123', 0, 0, 'VERIFIED', 'USER_VERIFICATION', now() - interval '301' second, now());

pom.xml

@@ -99,7 +99,7 @@
         <powerauth-restful-integration.version>1.5.0-SNAPSHOT</powerauth-restful-integration.version>
         <powerauth-push.version>1.5.0-SNAPSHOT</powerauth-push.version>
 
-        <bcprov-jdk18on.version>1.73</bcprov-jdk18on.version>
+        <bcprov-jdk18on.version>1.74</bcprov-jdk18on.version>
     </properties>
 
     <dependencyManagement>