[master] Zil 5417 create a driver program to run zilliqa code base native for testing purposes

native/README.md

@@ -0,0 +1,48 @@
+# native
+
+Ziliiqa Native Driver
+
+Description
+
+Create a program that will generate the required environment to launch a native version of the Zilliqa blockchain that will run on a local or server machine environment.
+
+The Zilliqa binaries will be built using the build.sh script and will support native debugging at the source code level using local development tools of your choice.
+
+Initial version is to run on a single host but can be extended later to run across any hosts if required.
+
+The initial version is independent of any other Zilliqa technologies and has no dependency on any other Zilliqa toolsets.
+
+The driver supports both the constants.xml configuration format and can utilize additional contract accounts as defined in isolated-server-accounts.json
+
+To run the tool simply run the native.py with the following arguments or variations of:
+
+-n 6 -d 5 -l 1--port 30303 --websocket= --transaction-sender=0
+--ds-guard=4
+--shard-guard=0
+--bucket=zilliqa-devnet
+--origin-server=/<yourdir>/dev/native/tmp
+--multiplier-fanout=1
+--out-dir=/<yourdir>/dev/native/tmp
+--build-dir=/<yourdir>/Zilliqa/build
+--skip-non-guard-ds
+--isolated-server-accounts
+
+A simple startup script
+
+Startall.sh is provided as a first cut for launching the initial version
+
+
+Prerequisites.
+
+Create a read/write directory /etc/zilliqa on your local machine (Annoying but hardcoded entry in zilliqa source)
+
+Out-dir is the path where the filesystem with binaries will be created, you need to run a web server there.   python3 -m http.server
+
+Currently you need to run a localstack instance.
+
+You can install localstack directly on your machine and start with localstack start
+
+
+TODO
+
+Fixup the hardhcoded path in the start.sh command, i usually run it inside by dir, I hope this is the last remaining defect with paths.

native/native.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Zilliqa
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import shutil
+
+from natlib.data import Data
+from natlib.lib import (is_seedprv, \
+                        generate_nodes, str2fanout, str2uints, str2lookup)
+
+LOOKUP_TYPES = ('lookup', 'seedpub', 'seedprv')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='initialize zilliqa')
+
+    parser.add_argument('--pod-name', help='pod name (TESTNET-TYPE-INDEX)')
+    parser.add_argument('--type', choices=['normal', 'lookup', 'new', 'seedprv'],
+                        help='node type (normal, lookup, new, seedprv)')
+    parser.add_argument('--index', type=int, help='node index')
+    parser.add_argument('--metadata-url', help='the base URL for metadata')
+    parser.add_argument('-n', type=int, help='number of all nodes', required=True)
+    parser.add_argument('-d', type=int, help='number of ds nodes', required=True)
+    parser.add_argument('-l', type=int, help='number of lookup nodes', required=True)
+    parser.add_argument('--ds-guard', type=int, default=0, help='number of ds guards')
+    parser.add_argument('--shard-guard', type=int, default=0, help='number of shard guards')
+    parser.add_argument('--skip-non-guard-ds', action='store_true', default=False,
+                        help='do not create pods for non-guard DS nodes')
+    parser.add_argument('--out-dir', help='The output directory')
+    parser.add_argument('--build-dir', help='The build directory')
+    parser.add_argument('--block-number-size', default=64, type=int, choices=[64, 256],
+                        help='block number size (64-bit, 256-bit)')
+    # TODO: --seed-multiplier is redudant when --multiplier-fanout is used
+    parser.add_argument('--seed-multiplier', action='store_true', default=True, help='Support seed multiplier')
+    parser.add_argument('--multiplier-fanout', type=str2fanout, help='the multiplier fanouts')
+    parser.add_argument('--max-recovery', default='2', help='Max tries for recovering zilliqa node')
+    parser.add_argument('--resume', action='store_true', help='Resume from persistent storage')
+    parser.add_argument('--recover-from-testnet', action='store_true', help='Recover all nodes from persistent storage')
+    parser.add_argument('--transaction-sender', default='0', type=str2uints, metavar='N',
+                        help='List of lookup indices that send testing transactions')
+    parser.add_argument('--origin-server', help='set external origin server instead using in-cluster one')
+    parser.add_argument('--lookup-dns-domain', help='the DNS name for lookup, multiplier, and seedpub nodes')
+    parser.add_argument('--log-path', help='Set customized log path')
+    parser.add_argument('--bucket', help='Set bucket name')
+    parser.add_argument('--hosted-zone-id', help='hosted zone ID')
+    parser.add_argument('--txnsbackup', action='store_true', help='Enable storing txns backup to S3')
+    parser.add_argument('--websocket', type=str2lookup, metavar='TYPE1,TYPE2',
+                        help='enable websocket for lookup server of TYPE, can be any of {}'.format(LOOKUP_TYPES))
+
+    group0 = parser.add_argument_group('Host Network Mode')
+    group0.add_argument('--port', type=int, help='port for zilliqa application')
+
+    group1 = parser.add_argument_group('Local Testing Mode', 'for testing and debugging init.py')
+    group1.add_argument('--testing', action='store_true', help='enable local testing mode')
+
+    group2 = parser.add_argument_group('Private keys cleanup Mode',
+                                       'for cleaning up non-self private keys before running the working container')
+    group2.add_argument('--cleanup-private-keys', action='store_true', help='clean up non-self private keys')
+
+    group3 = parser.add_argument_group('Native mode settings', 'for defining how native will behave')
+    group3.add_argument('--isolated-server-accounts', action='store_true',
+                        help='use the isolated server accounts file from zilliqa directory')
+
+    args = parser.parse_args()
+
+    # Inititalise the data object
+    data = Data()
+
+    # The origin server stores the network information
+    if args.origin_server is not None:
+        # if --origin-server is set, use it first
+        data.origin_server = args.origin_server
+    elif args.metadata_url is not None:
+        # metadata dns record used to be created with nginx ingress installation, not now
+        # If --metadata-url is set, use the internal one of origin
+        data.origin_server = '{}/origin/{}'.format(args.metadata_url, args.testnet)
+        # origin_server = args.metadata_url
+    else:
+        # This will not be used in the native implementation
+        data.origin_server = ""
+        print("We currently cannot start without something in origin server")
+        return 0
+
+    if args.isolated_server_accounts is not None:
+        # Copy into configmap
+        try:
+            shutil.copyfile("../isolated-server-accounts.json",
+                            os.path.join("/etc/zilliqa", "isolated-server-accounts.json"))
+        except:
+            print("Failed to copy isolated-server-accounts.json")
+            return 0
+
+    # The following is the same as the original implementation, except it generates keys rather than
+    # using the secrets directory
+
+    if not data.get_ips_list_from_pseudo_origin(args):
+        print("Generating data from the command line arguments failed")
+        return 0
+
+    args.restart = False
+
+    try:
+        os.mkdir(args.out_dir)
+    except FileExistsError:
+        pass
+
+    cwd = os.getcwd()
+
+    generate_nodes(args, data, 'lookup', 0, args.l)
+    generate_nodes(args, data, 'dsguard', 0, args.ds_guard)
+    generate_nodes(args, data, 'normal', 0, args.d - args.ds_guard)
+    generate_nodes(args, data, 'multiplier', 0, len(data.get_multiplier()))
+    generate_nodes(args, data, 'seedpub', 0, sum(args.multiplier_fanout))
+
+    return 0
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    main()

native/natlib/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Zilliqa
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.

native/natlib/data.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Zilliqa
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from typing import List
+from pyzil.crypto import zilkey
+
+# pod map for nodes of type normal and/or dsguard
+#
+# notations:
+# * N -> normal pod
+# * D -> dsguard pod
+# * d -> args.d
+# * n -> args.n
+# * dg -> args.ds_guard
+# * sg -> args.shard_guard
+
+# --------------------------------------------------------------------------------------------------------------------------
+# | line # in keys.txt | guard mode disabled  | guard mode enabled  | guard mode, non-guard DS skipped | is_* == True
+# --------------------------------------------------------------------------------------------------------------------------
+# | 0                  | N 0                  | D 0                 | D 0                              | is_ds, is_ds_guard
+# | 1                  | N 1                  | D 1                 | D 1                              | is_ds, is_ds_guard
+# | 2                  | N 2                  | D 2                 | D 2                              | is_ds, is_ds_guard
+# | ...                | ...                  | ...                 | ...                              | is_ds, is_ds_guard
+# | (dg - 1)           | N (dg - 1)           | D (dg - 1)          | D (dg - 1)                       | is_ds, is_ds_guard
+# --------------------------------------------------------------------------------------------------------------------------
+# | (dg)               | N (dg)               | N 0                 | skipped                          | is_ds,
+# | (dg + 1)           | N (dg + 1)           | N 1                 | skipped                          | is_ds,
+# | ...                | ...                  | ...                 | ...                              | is_ds,
+# | (d - 1)            | N (d - 1)            | N (d - dg - 1)      | skipped                          | is_ds,
+# --------------------------------------------------------------------------------------------------------------------------
+# | (d)                | N (d)                | N (d - dg)          | N 0                              | is_non_ds, is_shard_guard
+# | d + 1              | N (d + 1)            | N (d - dg + 1)      | N 1                              | is_non_ds, is_shard_guard
+# | ...                | ...                  | ...                 | ...                              | is_non_ds, is_shard_guard
+# | d + sg - 1         | N (d + sg - 1)       | N (d - dg + sg - 1) | N (sg - 1)                       | is_non_ds, is_shard_guard
+# --------------------------------------------------------------------------------------------------------------------------
+# | d + sg             | N (d + sg)           | N (d - dg + sg)     | N (sg)                           | is_non_ds,
+# | d + sg + 1         | N (d + sg + 1)       | N (d - dg + sg + 1) | N (sg + 1)                       | is_non_ds,
+# | ...                | ...                  | ...                 | ...                              | is_non_ds,
+# | (n - 1)            | N (n - 1)            | N (n - dg - 1)      | N (n - d - 1)                    | is_non_ds,
+
+
+class Data:
+
+    IP = 0
+    PORT = 1
+    PUBLIC = 2
+
+    def __init__(self) -> object:
+        self.normal_ips_from_origin = []
+        self.lookup_ips_from_origin = []
+        self.multiplier_ips_from_origin = []
+        self.seedpub_ips_from_origin = []
+        self.guard_ips_from_origin = []
+        self.origin_server = ""
+
+    def get_normal(self):
+        return self.normal_ips_from_origin
+
+    def get_lookup(self) -> List[object]:
+        return self.lookup_ips_from_origin
+
+    def get_multiplier(self):
+        return self.multiplier_ips_from_origin
+
+    def get_seedpub(self):
+        return self.seedpub_ips_from_origin
+
+    def get_guard(self):
+        return self.guard_ips_from_origin
+
+    def __str__(self):
+        return f"normal: {self.normal_ips_from_origin}\nlookup: {self.lookup_ips_from_origin}\nmultiplier: {self.multiplier_ips_from_origin}\nSeedpub:{self.seedpub_ips_from_origin}\nGaurd:{self.gaurd_ips_from_origin}\n"
+
+    def genkey(self, num) -> List[str]:
+        keypairs = []
+        for _ in range(num):
+            key = zilkey.ZilKey.generate_new()
+            publicKey, privateKey = key.keypair_str
+            keypairs.append(' '.join([publicKey.upper(), privateKey.upper()]))
+        keypairs.sort()
+        return keypairs
+
+    def get_ips_list_from_pseudo_origin(self, arg) -> bool:
+        try:
+            ''' Allocate the ports sequentitally '''
+            self.normal_port = arg.port
+            self.guard_port = self.normal_port + (((arg.n - arg.d)*5) + 1)
+            self.lookup_port = self.guard_port + (arg.ds_guard*5) + 1
+            self.seedpub_port = self.lookup_port + (arg.l*5) + 1
+            self.multiplier_port = self.seedpub_port + (sum(arg.multiplier_fanout)*5) + 1
+            self.nextfreeport = self.multiplier_port + (len(arg.multiplier_fanout)*5) + 1
+            self.my_ip = "192.168.0.98"
+
+        except Exception as _:
+            print("We suspect the arguments are suspect")
+            return False
+        try:
+            # lookup
+
+            self.lookup_ips_from_origin = list(
+                zip([str(self.my_ip)] * arg.l, range(self.lookup_port, self.lookup_port + arg.l, 5),
+                    self.genkey(arg.l)))
+            # seedpub
+            self.seedpub_ips_from_origin = list(zip([str(self.my_ip)] * sum(arg.multiplier_fanout),
+                                                    range(self.seedpub_port,
+                                                          self.seedpub_port + sum(arg.multiplier_fanout), 5),
+                                                    self.genkey(sum(arg.multiplier_fanout))))
+            # multiplier
+            self.multiplier_ips_from_origin = list(zip([str(self.my_ip)] * len(arg.multiplier_fanout),
+                                                       range(self.multiplier_port,
+                                                             self.multiplier_port + len(arg.multiplier_fanout), 5),
+                                                       self.genkey(len(arg.multiplier_fanout))))
+
+            # now normals
+
+            keypairs = self.genkey(arg.n)
+
+            self.guard_ips_from_origin = list(zip([str(self.my_ip)] * arg.ds_guard,
+                                                  range(self.guard_port,
+                                                        self.guard_port + (5 * arg.ds_guard), 5),
+                                                  keypairs[0:arg.ds_guard]))
+
+            keys = keypairs[arg.d:arg.n]
+            ports = range(self.normal_port, self.normal_port + arg.n - arg.d, 5)
+            ips = [str(self.my_ip)] * (arg.n - arg.ds_guard)
+            self.normal_ips_from_origin = list(zip(ips, ports, keys))
+
+        except Exception as _:
+            print("We suspect the arguments are suspect")
+            return False
+
+        return True