[master] ZIL-5403: check if a LevelDB object is null or not

scripts/localdev.py

@@ -343,15 +343,14 @@ def wait_for_helm_pod(config, pod_partial_name):
 
     return pod_name
 
-def localstack_up(config):
+def localstack_up(config, bucket_name):
     """ Let helm deploy localstack """
     run_or_die(config, ["helm", "upgrade", "--install", "localstack", "localstack/localstack"])
     localstack_pod_name = wait_for_helm_pod(config, "localstack-")
 
     # Port forward localstack so we can talk to it
     run_or_die(config, ["kubectl", "port-forward", "deployment/localstack", "4566:4566"], in_background=True)
 
-    bucket_name = 'zilliqa-devnet'
     run_or_die(config, ['kubectl', 'exec', '-it', localstack_pod_name, '--', 'awslocal', 's3', 'mb', f's3://{bucket_name}'])
     run_or_die(config, ['kubectl', 'exec', '-it', localstack_pod_name, '--', 'awslocal', 's3', 'mb', f's3://tempo'])
 
@@ -427,6 +426,10 @@ def prometheus_up(config, testnet_name, count = 8):
             ips.append(pod_ip)
 
         if len(ips) != count:
+            # If for some reason you see the following line repeating indefinitely when
+            # bringing up a devnet, it's most likely because the count is incorrect either
+            # due to a change in write_testnet_configuration or in the testnet repo (perhaps
+            # someone added/removed pods?).
             print(f"Waiting for all pods to be assigned an IP...")
             time.sleep(2)
         else:
@@ -507,12 +510,13 @@ def tempo_down(config):
               default=True,
               show_default=True,
               help="Use isolated_server_accounts.json to create accounts when zilliqa is up")
+@click.option("--bucket-name", help="The name of the bucket", default="zilliqa-devnet")
 @click.option("--persistence", help="A persistence directory to start the network with. Has no effect without also passing `--key-file`.")
 @click.option("--key-file", help="A `.tar.gz` generated by `./testnet.sh back-up auto` containing the keys used to start this network. Has no effect without also passing `--persistence`.")
 @click.option("--monitoring", help="Start monitoring - when set to false, skips grafana, prometheus, and tempo", default=True,show_default=True)
 @click.option("--desk", is_flag=True, default=False, help="use a small testnet configuration")
 @click.option("--chain-id", help="Set the chain id", default=None)
-def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id, desk):
+def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id, desk):
     """
     Build Zilliqa (via a process equivalent to the build-zilliqa & build-scilla commands), write configuration files for a
     testnet named localdev, run `localdev/config.sh up`, and start a proxy to allow the user to monitor traffic
@@ -524,15 +528,15 @@ def up_cmd(ctx, driver, zilliqa_image, testnet_name, isolated_server_accounts, p
     else:
         adjust_config(config, driver)
 
-    up(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id = chain_id,desk=desk)
+    up(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id = chain_id, desk=desk)
 
-def up(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, monitoring, chain_id, desk):
+def up(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, monitoring, chain_id, desk):
     minikube = get_minikube_ip(config)
-    write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, chain_id, desk)
-    localstack_up(config)
+    write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, chain_id, desk)
+    localstack_up(config, bucket_name)
     if monitoring:
         grafana_up(config, testnet_name)
-    start_testnet(config, testnet_name, persistence)
+    start_testnet(config, testnet_name, persistence, bucket_name)
     if monitoring:
         prometheus_up(config, testnet_name)
         tempo_up(config, testnet_name)
@@ -657,7 +661,7 @@ def isolated(config, enable_evm = True, block_time_ms = None, chain_id = None):
     # except:
     #     pass
 
-def start_testnet(config, testnet_name, persistence):
+def start_testnet(config, testnet_name, persistence, bucket_name):
     run_or_die(config, ["./testnet.sh", "up"], in_dir=os.path.join(TESTNET_DIR, testnet_name))
 
     if persistence is not None:
@@ -675,7 +679,6 @@ def aws(cmd):
 
         # Copy persistence to S3 in localstack. Each of the subdirectories in S3 are meant to contain only a subset of
         # persistence, but we choose to just copy everything into everywhere.
-        bucket_name = "zilliqa-devnet"
         aws(["s3", "sync", f"{persistence}/", f"s3://{bucket_name}/blockchain-data/{testnet_name}/"])
         aws(["s3", "sync", f"{persistence}/", f"s3://{bucket_name}/incremental/localdev/persistence/"])
         aws(["s3", "cp", f"_{testnet_name}/.currentTxBlk", f"s3://{bucket_name}/incremental/{testnet_name}/"])
@@ -731,14 +734,19 @@ def stop_testnet(config, testnet_name):
     # tediously, testnet.sh has a habit of returning non-zero error codes when eg. the testnet has already been destroyed :-(
     run_or_die(config, ["sh", "-c", "echo localdev | ./testnet.sh down"], in_dir=os.path.join(TESTNET_DIR, testnet_name), allow_failure = True)
 
-def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, persistence, key_file, chain_id,desk):
+def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_server_accounts, bucket_name, persistence, key_file, chain_id, desk):
     instance_dir = os.path.join(TESTNET_DIR, testnet_name)
     minikube_ip = get_minikube_ip(config)
 
     if os.path.exists(instance_dir):
         print(f"Removing old testnet configuration ..")
         shutil.rmtree(instance_dir)
     print(f"Generating testnet configuration .. ")
+
+    # IMPORTANT: if you change any of the flags make sure to update the count returned
+    #            by this function so that prometheus_up knows how many pods to wait for.
+    # 20 normal pods (i.e. 5 ds + 15 non-ds), 1 lookup, 2 seedpubs, 1 seedprv.
+    count = 20 + 1 + 2 + 1
     if desk:
         cmd = ["./bootstrap.py", testnet_name, "--clusters", "minikube", "--constants-from-file",
            os.path.join(ZILLIQA_DIR, "constants.xml"),
@@ -752,6 +760,7 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se
            "--host-network", "false",
            "--https", "localdomain",
            "--seed-multiplier", "true",
+           "--bucket", bucket_name,
            "--localstack", "true"]
     else:
         cmd = ["./bootstrap.py", testnet_name, "--clusters", "minikube", "--constants-from-file",
@@ -772,9 +781,7 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se
     cmd = cmd + ([ "--isolated-server-accounts", os.path.join(ZILLIQA_DIR, "isolated-server-accounts.json") ] if isolated_server_accounts else [])
     cmd = cmd + [ "-f" ]
     if persistence is not None and key_file is not None:
-        bucket_name = "zilliqa-devnet"
         cmd.extend([
-            "--bucket", bucket_name,
             "--recover-from-s3", f"s3://{bucket_name}/persistence/{testnet_name}-persistence.tar.gz",
             "--recover-key-files", key_file,
         ])
@@ -796,6 +803,8 @@ def write_testnet_configuration(config, zilliqa_image, testnet_name, isolated_se
     with open(constants_xml_target_path, 'w') as f:
         f.write(output_config)
 
+    return count 
+
 def kill_mitmweb(config, pidfile_name):
     pidfile = Pidfile(config, pidfile_name)
     pid = pidfile.get()
@@ -891,7 +900,7 @@ def build_native_to_workspace(config):
     build_env['SCILLA_REPO_ROOT'] = SCILLA_DIR
     # Let's start off by building Scilla, in case it breaks.
     run_or_die(config, ["make"], in_dir = SCILLA_DIR, env = build_env)
-    run_or_die(config, ["./build.sh"], in_dir = ZILLIQA_DIR)
+    run_or_die(config, ["./build.sh", "ninja", "debug"], in_dir = ZILLIQA_DIR)
     run_or_die(config, ["cargo", "build", "--release", "--package", "evm-ds"], in_dir =
                os.path.join(ZILLIQA_DIR, "evm-ds"))
     # OK. That worked. Now copy the relevant bits to our workspace
@@ -1114,24 +1123,69 @@ def build_zilliqa(config, driver, scilla_image, tag):
                capture_output = False)
     return image_name
 
+def build_native_zilliqa(config):
+    build_env = os.environ.copy()
+    build_env.update(config.default_env)
+
+    if not build_env.get("VCPKG_ROOT"):
+        raise GiveUp("Environment variable VCPKG_ROOT must be defined to point to vcpkg")
+
+    #  vcpkg_triplet = run_or_die(config, ["scripts/vcpkg_triplet.sh"], in_dir = ZILLIQA_DIR, env = build_env,
+               #  capture_output = True).decode('utf-8')
+
+    # Cleanup
+    build_dir = os.path.join(ZILLIQA_DIR, ".build.vcpkg")
+    shutil.rmtree(os.path.join(build_dir, "vcpkg_installed"), ignore_errors = True)
+    shutil.rmtree(os.path.join(build_dir, "CMakeFiles"), ignore_errors = True)
+    try:
+        os.remove(os.path.join(build_dir, "CMakeCache.txt"))
+    except:
+        pass
+
+    run_or_die(config, ["./build.sh", "ninja", "debug"], in_dir = ZILLIQA_DIR)
+    run_or_die(config, ["cargo", "build", "--release", "--package", "evm-ds"], in_dir =
+               os.path.join(ZILLIQA_DIR, "evm-ds"))
+
 @click.command("build-zilliqa")
 @click.option("--driver",
               required=True,
               default=default_driver(),
               show_default=True,
               help="The minikube driver to use")
 @click.option("--scilla-image",
-              required=True,
+              required=False,
               help="the scilla image to use when building the zilliqa image (i.e. scilla:<tag>)")
 @click.option("--tag",
               help="The zilliqa image tag. Will be generated if not given.")
+@click.option("--native",
+              is_flag=True,
+              default=False,
+              show_default=True,
+              help="Build Scilla natively.")
 @click.pass_context
-def build_zilliqa_cmd(ctx, driver, scilla_image, tag):
+def build_zilliqa_cmd(ctx, driver, scilla_image, tag, native):
     """
     Builds a zilliqa image.
     """
+    if native:
+        if tag:
+            raise GiveUp("--native and --tag can't be specified together")
+        elif ctx.params["driver"]:
+            print("ignoring --driver since --native is specified; building natively...")
+    else:
+        if not scilla_image:
+            raise GiveUp("--scilla-image must be specified")
+
+        if not driver:
+            driver = default_driver()
+        ctx.params["driver"] = driver
+
+
     config = get_config(ctx)
-    build_zilliqa(config, driver, scilla_image, tag)
+    if native:
+       build_native_zilliqa(config)
+    else:
+       build_zilliqa(config, driver, scilla_image, tag)
 
 def get_pod_names(config, node_type = None):
     cmd =  ["kubectl",