Add standalone client tests [PA-547]

.gitignore

@@ -14,4 +14,6 @@ venv/
 lib/
 src/simulator/__pycache__/
 src/__pycache__/
-m2/
+m2/
+.vscode
+myproject

README.md

@@ -96,19 +96,23 @@ Also contains pointers where to go next.
     - JDK 17 or newer
     - [Maven](https://maven.apache.org/download.cgi)
     - [Terraform](https://developer.hashicorp.com/terraform/downloads)
-    - [Python 3.8 or newer](https://www.python.org/downloads/)
+    - Python 3.8
     - Python3-pip
     - python3-gobject
     - rsync
 
-3. Install Python libraries:
+3. Create a virtual environment, activate it and install dependencies
 
     ```shell
-    pip3 install -U ansible pyyaml matplotlib signal-processing-algorithms pandas plotly boto3
-    ```
+   python3.8 -m venv venv && source ./venv/bin/activate
+   ```
+
+ ```shell
+   pip install -U -r requirements.txt
+   ```
 
    `signal-processing-algorithms` is only needed when you are going to do performance regression testing. The library is
-   used for change point detection.
+   used for change point detection:
 
 
 4. Build Simulator:

java/drivers/driver-hazelcast4plus/conf/configure.py

@@ -12,7 +12,7 @@ def _configure_hazelcast_xml(nodes, args: DriverConfigureArgs, is_lite_member: b
         if not os.path.exists(src_file):
             raise Exception(f"Could not find hazelcast.xml with path [{src_file}]")
     else:
-        src_file = find_driver_config_file(_get_driver(args), "hazelcast.xml")
+        src_file = find_driver_config_file(args.driver, "hazelcast.xml")
 
     config = read_file(src_file)
 
@@ -49,7 +49,7 @@ def _configure_client_hazelcast_xml(nodes, args: DriverConfigureArgs):
         if not os.path.exists(src_file):
             raise Exception(f"Could not find client-hazelcast.xml with path [{src_file}]")
     else:
-        src_file = find_driver_config_file(_get_driver(args), "client-hazelcast.xml")
+        src_file = find_driver_config_file(args.driver, "client-hazelcast.xml")
 
     config = read_file(src_file)
 
@@ -63,13 +63,13 @@ def _configure_client_hazelcast_xml(nodes, args: DriverConfigureArgs):
 
 
 def _configure_log4j_xml(args: DriverConfigureArgs):
-    driver = _get_driver(args)
+    driver = args.driver
     log4j_xml = read_file(find_driver_config_file(driver, "log4j.xml"))
     args.coordinator_params['file:log4j.xml'] = log4j_xml
 
 
 def _configure_worker_sh(args: DriverConfigureArgs):
-    driver = _get_driver(args)
+    driver = args.driver
 
     worker_sh_path = args.test.get('worker_sh')
     if worker_sh_path is None:
@@ -79,14 +79,6 @@ def _configure_worker_sh(args: DriverConfigureArgs):
 
     args.coordinator_params['file:worker.sh'] = worker_sh
 
-
-def _get_driver(args: DriverConfigureArgs):
-    driver = args.test.get('driver')
-    if driver is None:
-        raise Exception(f"Could not find 'driver' in test {args.test['name']}")
-    return driver
-
-
 def exec(args: DriverConfigureArgs):
     info("Configure")
 

java/drivers/driver-hazelcast4plus/conf/install.py

@@ -116,7 +116,7 @@ def _get_force_download_from_maven_repo(version_spec:str):
 
 
 def _upload_hazelcast_jars(args:DriverInstallArgs, hosts):
-    driver = _get_driver(args)
+    driver = args.driver
     is_enterprise = _get_is_enterprise(driver)
     version_spec = args.test.get('version', 'maven=5.4.0')
     version = _get_version(version_spec)
@@ -141,10 +141,6 @@ def _upload_hazelcast_jars(args:DriverInstallArgs, hosts):
     info(f"Uploading Hazelcast jars: done")
 
 
-def _get_driver(args):
-    return args.test.get('driver')
-
-
 def exec(args:DriverInstallArgs):
     info("Install")
 
@@ -157,7 +153,7 @@ def exec(args:DriverInstallArgs):
     loadgenerator_hosts = args.test.get("loadgenerator_hosts")
     hosts.extend(load_hosts(inventory_path=args.inventory_path, host_pattern=loadgenerator_hosts))
 
-    upload_driver(_get_driver(args), hosts)
+    upload_driver(args.driver, hosts)
     _upload_hazelcast_jars(args, hosts)
 
     info("Install: done")

java/drivers/driver-standaloneclient4plus/conf/client-hazelcast.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<hazelcast-client
+        xsi:schemaLocation="http://www.hazelcast.com/schema/client-config
+            http://www.hazelcast.com/schema/config/hazelcast-client-config-5.0.xsd"
+        xmlns="http://www.hazelcast.com/schema/client-config"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+    <cluster-name>workers</cluster-name>
+
+    <network>
+        <cluster-members>
+            <!--MEMBERS-->
+        </cluster-members>
+    </network>
+
+    <properties>
+        <property name="hazelcast.logging.type">log4j2</property>
+    </properties>
+
+    <serialization>
+        <data-serializable-factories>
+            <data-serializable-factory factory-id="4000">
+                com.hazelcast.simulator.tests.map.domain.IdentifiedDataSerializableObjectFactory
+            </data-serializable-factory>
+        </data-serializable-factories>
+
+        <portable-version>1</portable-version>
+        <portable-factories>
+            <portable-factory factory-id="10000001">com.hazelcast.simulator.tests.map.domain.PortableObjectFactory</portable-factory>
+            <portable-factory factory-id="10000002">com.hazelcast.simulator.tests.map.helpers.ComplexDomainObjectPortableFactory</portable-factory>
+        </portable-factories>
+    </serialization>
+</hazelcast-client>

java/drivers/driver-standaloneclient4plus/conf/configure.py

@@ -0,0 +1,104 @@
+import os.path
+
+from inventory import load_hosts
+from simulator.driver import find_driver_config_file, DriverConfigureArgs
+from simulator.log import info
+from simulator.util import read_file
+
+
+def _configure_hazelcast_xml(nodes, args: DriverConfigureArgs, is_lite_member: bool):
+    src_file = args.test.get("hazelcast_xml")
+    if src_file is not None:
+        if not os.path.exists(src_file):
+            raise Exception(f"Could not find hazelcast.xml with path [{src_file}]")
+    else:
+        src_file = find_driver_config_file(args.driver, "hazelcast.xml")
+
+    config = read_file(src_file)
+
+    # configure <!--LICENSE-KEY-->
+    license_key = args.test.get("license_key")
+    if license_key is not None:
+        license_key_config = f"<license-key>{license_key}</license-key>"
+        config = config.replace("<!--LICENSE-KEY-->", license_key_config)
+
+    # configure <!--LITE_MEMBER_CONFIG-->
+    if is_lite_member:
+        lite_member_config = "<lite-member enabled=\"true\"/>"
+    else:
+        lite_member_config = "<lite-member enabled=\"false\"/>"
+
+    config = config.replace("<!--LITE_MEMBER_CONFIG-->", lite_member_config)
+
+    # configure <!--MEMBERS-->
+    members_config = ""
+    member_port = args.test.get('member_port', '5701')
+    for host in nodes:
+        members_config = f"{members_config}<member>{host['private_ip']}:{member_port}</member>"
+    config = config.replace("<!--MEMBERS-->", members_config)
+
+    if is_lite_member:
+        args.coordinator_params['file:litemember-hazelcast.xml'] = config
+    else:
+        args.coordinator_params['file:hazelcast.xml'] = config
+
+
+def _configure_client_hazelcast_xml(nodes, args: DriverConfigureArgs):
+    src_file = args.test.get("client_hazelcast_xml")
+    if src_file is not None:
+        if not os.path.exists(src_file):
+            raise Exception(f"Could not find client-hazelcast.xml with path [{src_file}]")
+    else:
+        src_file = find_driver_config_file(args.driver, "client-hazelcast.xml")
+
+    config = read_file(src_file)
+
+    members_config = ""
+    member_port = args.test.get('member_port', '5701')
+    for host in nodes:
+        members_config = f"{members_config}<address>{host['private_ip']}:{member_port}</address>"
+
+    config = config.replace("<!--MEMBERS-->", members_config)
+    args.coordinator_params['file:client-hazelcast.xml'] = config
+
+
+def _configure_log4j_xml(args: DriverConfigureArgs):
+    driver = args.driver
+    log4j_xml = read_file(find_driver_config_file(driver, "log4j.xml"))
+    args.coordinator_params['file:log4j.xml'] = log4j_xml
+
+
+def _configure_worker_sh(args: DriverConfigureArgs):
+    driver = args.driver
+
+    worker_sh_path = args.test.get('worker_sh')
+    if worker_sh_path is None:
+        worker_sh = read_file(find_driver_config_file(driver, "worker.sh"))
+    else:
+        worker_sh = read_file(worker_sh_path)
+
+    args.coordinator_params['file:worker.sh'] = worker_sh
+
+
+def exec(args: DriverConfigureArgs):
+    info("Configure")
+
+    nodes_pattern = args.test.get("node_hosts")
+    nodes = load_hosts(inventory_path=args.inventory_path, host_pattern=nodes_pattern)
+
+    node_count = args.test.get("node_count")
+    if node_count is not None:
+        # For Hazelcast we need to have workers for both clients (load generators) and
+        # nodes (members). By setting the NODE_WORKER_COUNT, we control the number of
+        # passive member workers that are started.
+        # There are also drivers like Redis OS that doesn't require any workers for the
+        # server, so NODE_WORKER_COUNT should be zero (which is the default).
+        args.coordinator_params['NODE_WORKER_COUNT'] = node_count
+
+    _configure_log4j_xml(args)
+    _configure_worker_sh(args)
+    _configure_hazelcast_xml(nodes, args, False)
+    _configure_hazelcast_xml(nodes, args, True)
+    _configure_client_hazelcast_xml(nodes, args)
+
+    info("Configure: done")

java/drivers/driver-standaloneclient4plus/conf/hazelcast.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<hazelcast xmlns="http://www.hazelcast.com/schema/config"
+           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+           xsi:schemaLocation="http://www.hazelcast.com/schema/config
+           http://www.hazelcast.com/schema/config/hazelcast-config-5.0.xsd">
+
+    <cluster-name>workers</cluster-name>
+
+    <!--LITE_MEMBER_CONFIG-->
+
+    <network>
+        <port port-count="200">5701</port>
+
+        <join>
+            <multicast enabled="false"/>
+            <tcp-ip enabled="true">
+                <!--MEMBERS-->
+            </tcp-ip>
+        </join>
+    </network>
+
+    <properties>
+        <property name="hazelcast.phone.home.enabled">false</property>
+    </properties>
+
+    <!--LICENSE-KEY-->
+
+    <cache name="*">
+    </cache>
+</hazelcast>
+