Merge pull request #1 from DaanSelen/dev

Version v0.1.1 added variable support!

README.md

@@ -13,68 +13,94 @@ Then make a yaml with a target and some commands! See below examples as a guidel
 To install, follow the following commands:<br>
 
 ```shell
+git clone https://github.com/daanselen/meshbook
 cd ./meshbook
 python3 -m venv ./venv
 source ./venv/bin/activate
-pip3 install -r requirements.txt
+pip3 install -r ./meshbook/requirements.txt
 ```
 Then you can use meshbook, for example:
 ```shell
-python3 meshbook.py -pb examples/ping.yaml
+python3 ./meshbook/meshbook.py -pb examples/echo.yaml
 ```
 
 # Example:
 
 For the example, I used the following yaml file:
 
-```yaml
+The below group: `Temp-Agents` has four devices, of which one is offline.<br>
----
-name: Ping a single Point
-company: Temp-Agents
-tasks:
-  - name: Ping Cloudflare
-    command: "ping 1.1.1.1 -c 4"
-```
-
-The above group: `Temp-Agents` has four devices, of which one is offline.<br>
 You can expand the command chain as follows:<br>
 
 ```yaml
 ---
 name: Ping Multiple Points
 company: Temp-Agents
+variables:
+  - name: host1
+    value: "1.1.1.1"
+  - name: host2
+    value: "ns.systemec.nl"
+  - name: command1
+    value: "ping"
+  - name: cmd_arguments
+    value: "-c 4"
 tasks:
-  - name: Ping Cloudflare
+  - name: Ping host1
-    command: "ping 1.1.1.1 -c 4"
+    command: "{{ command1 }} {{ host1 }} {{ cmd_arguments }}"
 
-  - name: Ping Google
+  - name: Ping host2
-    command: "ping 8.8.8.8 -c 4"
+    command: "{{ command1 }} {{ host2 }} {{ cmd_arguments }}"
 ```
 
 The following response it received when executing the first yaml of the above files.
 
 ```shell
-python3 meshbook.py -pb examples/ping.yaml -s
+python3 meshbook/meshbook.py -pb examples/variable_example.yaml -si
+-=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
+Running task: {'name': 'Ping host1', 'command': 'ping 1.1.1.1 -c 4'}
+-=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
+Current Batch: 1
+Current response number: 1
+Current Calculation: 1 % 3 = 1
+Current Batch: 1
+Current response number: 2
+Current Calculation: 2 % 3 = 2
+Current Batch: 1
+Current response number: 3
+Current Calculation: 3 % 3 = 0
+-=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
+Running task: {'name': 'Ping host2', 'command': 'ping ns.systemec.nl -c 4'}
+-=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
+Current Batch: 2
+Current response number: 4
+Current Calculation: 4 % 3 = 1
+Current Batch: 2
+Current response number: 5
+Current Calculation: 5 % 3 = 2
+Current Batch: 2
+Current response number: 6
+Current Calculation: 6 % 3 = 0
+-=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=--=-
 {
     "Batch 1": [
         {
             "action": "msg",
             "type": "runcommands",
-            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=59 time=6.88 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=59 time=6.50 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=59 time=6.46 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=59 time=6.51 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3005ms\nrtt min/avg/max/mdev = 6.460/6.588/6.879/0.169 ms\n",
+            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=59 time=6.70 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=59 time=6.51 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=59 time=6.51 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=59 time=6.52 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3005ms\nrtt min/avg/max/mdev = 6.508/6.558/6.697/0.080 ms\n",
             "responseid": "meshctrl",
             "nodeid": "MSI"
         },
         {
             "action": "msg",
             "type": "runcommands",
-            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=57 time=6.22 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=57 time=6.07 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=57 time=5.97 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=57 time=5.90 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3004ms\nrtt min/avg/max/mdev = 5.904/6.038/6.216/0.117 ms\n",
+            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=57 time=6.15 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=57 time=6.43 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=57 time=5.94 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=57 time=5.87 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3004ms\nrtt min/avg/max/mdev = 5.870/6.098/6.430/0.217 ms\n",
             "responseid": "meshctrl",
             "nodeid": "server"
         },
         {
             "action": "msg",
             "type": "runcommands",
-            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=59 time=6.83 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=59 time=6.64 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=59 time=6.65 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=59 time=6.53 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3005ms\nrtt min/avg/max/mdev = 6.534/6.664/6.834/0.108 ms\n",
+            "result": "PING 1.1.1.1 (1.1.1.1) 56(84) bytes of data.\n64 bytes from 1.1.1.1: icmp_seq=1 ttl=57 time=6.29 ms\n64 bytes from 1.1.1.1: icmp_seq=2 ttl=57 time=6.05 ms\n64 bytes from 1.1.1.1: icmp_seq=3 ttl=57 time=5.88 ms\n64 bytes from 1.1.1.1: icmp_seq=4 ttl=57 time=5.99 ms\n\n--- 1.1.1.1 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3005ms\nrtt min/avg/max/mdev = 5.875/6.050/6.286/0.150 ms\n",
             "responseid": "meshctrl",
             "nodeid": "raspberrypi5"
         }
@@ -83,27 +109,29 @@ python3 meshbook.py -pb examples/ping.yaml -s
         {
             "action": "msg",
             "type": "runcommands",
-            "result": "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=5.69 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=5.22 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=5.19 ms\n64 bytes from 8.8.8.8: icmp_seq=4 ttl=118 time=5.16 ms\n\n--- 8.8.8.8 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3004ms\nrtt min/avg/max/mdev = 5.161/5.315/5.694/0.219 ms\n",
+            "result": "PING ns.systemec.nl (89.20.90.102) 56(84) bytes of data.\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=1 ttl=60 time=1.45 ms\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=2 ttl=60 time=1.10 ms\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=3 ttl=60 time=1.12 ms\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=4 ttl=60 time=1.14 ms\n\n--- ns.systemec.nl ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3003ms\nrtt min/avg/max/mdev = 1.100/1.199/1.448/0.143 ms\n",
-            "responseid": "meshctrl",
-            "nodeid": "MSI"
-        },
-        {
-            "action": "msg",
-            "type": "runcommands",
-            "result": "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=5.65 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=5.28 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=5.25 ms\n64 bytes from 8.8.8.8: icmp_seq=4 ttl=118 time=5.25 ms\n\n--- 8.8.8.8 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3004ms\nrtt min/avg/max/mdev = 5.246/5.357/5.648/0.168 ms\n",
             "responseid": "meshctrl",
             "nodeid": "raspberrypi5"
         },
         {
             "action": "msg",
             "type": "runcommands",
-            "result": "PING 8.8.8.8 (8.8.8.8) 56(84) bytes of data.\n64 bytes from 8.8.8.8: icmp_seq=1 ttl=118 time=4.94 ms\n64 bytes from 8.8.8.8: icmp_seq=2 ttl=118 time=4.68 ms\n64 bytes from 8.8.8.8: icmp_seq=3 ttl=118 time=4.79 ms\n64 bytes from 8.8.8.8: icmp_seq=4 ttl=118 time=4.77 ms\n\n--- 8.8.8.8 ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3005ms\nrtt min/avg/max/mdev = 4.678/4.792/4.940/0.094 ms\n",
+            "result": "PING ns.systemec.nl (89.20.90.102) 56(84) bytes of data.\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=1 ttl=59 time=1.52 ms\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=2 ttl=59 time=1.26 ms\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=3 ttl=59 time=1.34 ms\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=4 ttl=59 time=1.27 ms\n\n--- ns.systemec.nl ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3006ms\nrtt min/avg/max/mdev = 1.255/1.345/1.523/0.107 ms\n",
             "responseid": "meshctrl",
             "nodeid": "server"
+        },
+        {
+            "action": "msg",
+            "type": "runcommands",
+            "result": "PING ns.systemec.nl (89.20.90.102) 56(84) bytes of data.\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=1 ttl=62 time=7.21 ms\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=2 ttl=62 time=0.346 ms\n64 bytes from ns.systemec.nl (89.20.90.102): icmp_seq=3 ttl=62 time=0.358 ms\n64 bytes from roma.systemec.nl (89.20.90.102): icmp_seq=4 ttl=62 time=0.336 ms\n\n--- ns.systemec.nl ping statistics ---\n4 packets transmitted, 4 received, 0% packet loss, time 3018ms\nrtt min/avg/max/mdev = 0.336/2.061/7.205/2.969 ms\n",
+            "responseid": "meshctrl",
+            "nodeid": "MSI"
         }
     ]
 }
+All tasks completed successfully: Expected 6 Received 6
 ```
+The above with `-si` is quite verbose. use `--help` to read about parameters.
 
 # Important Notice:
 

examples/apt_upgrade.yaml

@@ -0,0 +1,18 @@
+---
+name: Refresh the apt cache
+company: Temp-Agents
+variables:
+  - name: package_manager
+    value: "apt"
+tasks:
+  - name: refresh the cache
+    command: "{{ package_manager }} update"
+
+  - name: display available upgrades
+    command: "{{ package_manager }} list --upgradable"
+
+  - name: apply upgrades
+    command: "{{ package_manager }} upgrade -y"
+
+  - name: cleanup remaining packages
+    command: "{{ package_manager }} autoremove -y"

examples/echo.yaml

@@ -0,0 +1,11 @@
+---
+name: "Echo some text in the terminal of the device"
+device: raspberrypi5
+variables:
+  - name: var1
+    value: "Testing"
+  - name: var2
+    value: "with a slash!"
+tasks:
+  - name: Echo!
+    command: "echo {{ var1 }}/{{ var2 }}"

examples/refresh_aptcache.yaml

@@ -0,0 +1,12 @@
+---
+name: Refresh the apt cache
+company: Temp-Agents
+variables:
+  - name: package_manager
+    value: "apt"
+tasks:
+  - name: refresh the cache
+    command: "{{ package_manager }} update"
+
+  - name: display available upgrades
+    command: "{{ package_manager }} list --upgradable"

examples/variable_example.yaml

@@ -0,0 +1,18 @@
+---
+name: Ping Multiple Points
+company: Temp-Agents
+variables:
+  - name: host1
+    value: "1.1.1.1"
+  - name: host2
+    value: "ns.systemec.nl"
+  - name: command1
+    value: "ping"
+  - name: cmd_arguments
+    value: "-c 4"
+tasks:
+  - name: Ping host1
+    command: "{{ command1 }} {{ host1 }} {{ cmd_arguments }}"
+
+  - name: Ping host2
+    command: "{{ command1 }} {{ host2 }} {{ cmd_arguments }}"

meshbook/examples/apt_upgrade.yaml

@@ -1,16 +0,0 @@
----
-name: Refresh the apt cache
-company: Temp-Agents
-#device: MSI
-tasks:
-  - name: refresh the cache
-    command: "apt update"
-
-  - name: display available upgrades
-    command: "apt list --upgradable"
-
-  - name: apply upgrades
-    command: "apt upgrade -y"
-
-  - name: cleanup remaining packages
-    command: "apt autoremove -y"

meshbook/examples/ping.yaml

@@ -1,9 +0,0 @@
----
-name: Ping Multiple Points
-company: Temp-Agents
-tasks:
-  - name: Ping Cloudflare
-    command: "ping 1.1.1.1 -c 4"
-
-  - name: Ping Google
-    command: "ping 8.8.8.8 -c 4"

meshbook/examples/refresh_aptcache.yaml

@@ -1,9 +0,0 @@
----
-name: Refresh the apt cache
-company: Temp-Agents
-tasks:
-  - name: refresh the cache
-    command: "apt update"
-
-  - name: display available upgrades
-    command: "apt list --upgradable"

meshbook/meshbook.py

@@ -73,6 +73,20 @@ class MeshbookUtilities:
         with open(file_path, 'r') as file:
             return yaml.safe_load(file)
         
+    @staticmethod
+    def replace_placeholders(playbook) -> dict:
+        # Convert 'variables' to a dictionary for quick lookup
+        variables = {var["name"]: var["value"] for var in playbook.get("variables", [])}
+
+        # Traverse 'tasks' to replace placeholders
+        for task in playbook.get("tasks", []):
+            command = task.get("command", "")
+            for var_name, var_value in variables.items():
+                placeholder = f"{{{{ {var_name} }}}}"  # Create the placeholder string like "{{ host1 }}"
+                command = command.replace(placeholder, var_value)  # Update the command string
+            task["command"] = command  # Save the updated command string
+        return playbook
+
     @staticmethod
     def translate_nodeids(batches_dict, global_list) -> dict:
         for batch_name, items in batches_dict.items():
@@ -220,6 +234,7 @@ class MeshbookProcessor:
         """Processes messages received from the WebSocket."""
         global response_counter
         temp_responses_list = []
+        
         while True:
             message = await python_client.received_response_queue.get()
             action_type = message.get('action')
@@ -250,13 +265,12 @@ class MeshcallerActions:
     """Processes playbook actions."""
     
     @staticmethod
-    async def process_arguments(python_client: MeshbookWebsocket, playbook_path: str):
+    async def process_arguments(python_client: MeshbookWebsocket, playbook_yaml: dict):
         """Executes tasks defined in the playbook."""
         global response_counter, expected_responses, target_ids
 
         await basic_ready_state.wait()  # Wait for the basic data to be ready
 
-        playbook_yaml = MeshbookUtilities.read_yaml(playbook_path)
         target_ids = MeshbookUtilities.get_target_ids(
             company=playbook_yaml.get('company'),
             device=playbook_yaml.get('device')
@@ -308,9 +322,10 @@ class MeshcallerActions:
 
 async def main():
     parser = argparse.ArgumentParser(description="Process command-line arguments")
+    parser.add_argument("-pb", "--playbook", type=str, help="Path to the playbook file.", required=True)
+
     parser.add_argument("--conf", type=str, help="Path for the API configuration file (default: ./api.conf).")
     parser.add_argument("--nojson", action="store_true", help="Makes the program not output the JSON response data.")
-    parser.add_argument("-pb", "--playbook", type=str, help="Path to the playbook file.", required=True)
     parser.add_argument("-s", "--silent", action="store_true", help="Suppress terminal output.")
     parser.add_argument("-i", "--information", action="store_true", help="Add the calculations and other informational data to the output.")
 
@@ -328,7 +343,11 @@ async def main():
             credentials['password']
         ))
         processor_task = asyncio.create_task(processor.receive_processor(python_client))
-        await MeshcallerActions.process_arguments(python_client, args.playbook)
+
+        playbook_yaml = MeshbookUtilities.read_yaml(args.playbook)
+        translated_playbook = MeshbookUtilities.replace_placeholders(playbook_yaml)
+        await MeshcallerActions.process_arguments(python_client, translated_playbook)
+
         await asyncio.gather(websocket_task, processor_task)
         
     except ScriptEndTrigger as e: