create sleep times and random failure

consts/numerical_values.py

 HEXADECIMAL_WORD_LENGTH = 15
+
+MIN_SLEEP_TIME = 1
+MAX_SLEEP_TIME = 15
+
+FAILURE_PERCENTAGE = 0.05
\ No newline at end of file

controllers/duplicate_controller.py

@@ -10,6 +10,7 @@ from utils.database_utils import (get_resource, get_state, insert_command,
                                   insert_presence_bulk, insert_resource,
                                   update_state)
 from utils.http_utils import normalize_url
+from utils.time_utils import random_sleep
 
 duplicate_controller = Blueprint('duplicate_controller', __name__)
 api = Api(duplicate_controller)
@@ -22,6 +23,7 @@ class DuplicateResourceResource(Resource):
             return response_offline, 403
 
         update_state(Status.UPDATING, normalize_url(request.host_url))
+        random_sleep(fail_percentage=0)
 
         body = request.get_json()
         resource = body.get('resource')
@@ -53,6 +55,7 @@ class DuplicateCommandResource(Resource):
             return response_offline, 403
 
         update_state(Status.UPDATING, normalize_url(request.host_url))
+        random_sleep(fail_percentage=0)
 
         resource = get_resource(id)
         if resource is None:

controllers/resource_controller.py

@@ -8,6 +8,7 @@ from models.command import Command
 from utils.database_utils import (get_presence, get_resource, get_state,
                                   insert_command, update_state)
 from utils.http_utils import normalize_url, send_command_to_other_nodes
+from utils.time_utils import random_sleep
 
 resource_controller = Blueprint('resource_controller', __name__)
 api = Api(resource_controller)
@@ -30,6 +31,7 @@ class ResourceResource(Resource):
         node_address = normalize_url(request.host_url)
         node_address_intern = normalize_url(request.host_url, True)
         update_state(Status.UPDATING, node_address)
+        random_sleep(fail_percentage=0)
 
         request_command: Optional[str] = request.get_json().get('command')
         request_addresses: list[str] = request.get_json().get('addresses', None)
@@ -53,7 +55,7 @@ class ResourceResource(Resource):
         command = Command(request_command)
 
         # TODO: Potentiellement long, à mettre dans un thread ?
-        send_command_to_other_nodes(request_addresses, id, command)
+        send_command_to_other_nodes(request_addresses, id, command, source_address=node_address)
 
         insert_command(id, command)
         update_state(Status.ONLINE, node_address)
@@ -65,6 +67,7 @@ class ResourceResource(Resource):
         node_address = normalize_url(request.host_url)
         update_state(Status.UPDATING, node_address)
 
+        random_sleep(fail_percentage=0)
         request_addresses: list[str] = request.get_json().get('addresses', [])
         request_addresses = list(map(lambda x: normalize_url(x, True), request_addresses))
 
@@ -79,7 +82,7 @@ class ResourceResource(Resource):
         command = Command("DELETE")
 
         # TODO: Potentiellement long, à mettre dans un thread ?
-        send_command_to_other_nodes(request_addresses, id, command)
+        send_command_to_other_nodes(request_addresses, id, command, source_address=node_address)
 
         insert_command(id, command)
         update_state(Status.ONLINE, node_address)

controllers/resources_controller.py

@@ -10,6 +10,7 @@ from utils.database_utils import (get_resources, get_state, insert_command,
                                   insert_presence_bulk, insert_resource,
                                   update_state)
 from utils.http_utils import normalize_url, send_resource_to_other_nodes
+from utils.time_utils import random_sleep
 
 resources_controller = Blueprint('resources_controller', __name__)
 api = Api(resources_controller)
@@ -41,6 +42,7 @@ class ResourcesResource(Resource):
         presences.append(node_address)
         request_addresses = list(map(lambda x: normalize_url(x, True), request_addresses))
 
+        random_sleep(fail_percentage=0)
         if request_command is None:
             update_state(Status.ONLINE, node_address, {"message": "command is missing", "statusCode": 400})
             return {"message": "command is missing"}, 400
@@ -59,7 +61,7 @@ class ResourcesResource(Resource):
         )
 
         # TODO: Potentiellement long, à mettre dans un thread ?
-        send_resource_to_other_nodes(request_addresses, resource)
+        send_resource_to_other_nodes(request_addresses, resource, source_address=node_address)
 
         insert_resource(resource)
         insert_command(resource.id, resource.commands[0])

controllers/state_controller.py

@@ -10,6 +10,7 @@ from utils.database_utils import (clear_command_queue, clear_resource_queue,
                                   update_state)
 from utils.http_utils import normalize_url
 from utils.queue_utils import fetch_insert_queues
+from utils.time_utils import random_sleep
 
 state_controller = Blueprint('state_controller', __name__)
 api = Api(state_controller)
@@ -20,6 +21,7 @@ class StopNode(Resource):
         if get_state() == Status.OFFLINE:
             return {"message": "Node already stoped."}, 403
         else:
+            random_sleep(fail_percentage=0)
             node_address = normalize_url(request.host_url)
             update_state(Status.OFFLINE, node_address)
             return {"message": "Successfully stoped."}, 200

utils/http_utils.py

@@ -4,9 +4,12 @@ from urllib.parse import urlparse
 
 import requests
 
+from consts.status import Status
 from models.command import Command
 from models.resource import Resource
-from utils.database_utils import insert_command_queue, insert_resource_queue
+from utils.database_utils import (insert_command_queue, insert_resource_queue,
+                                  update_state)
+from utils.time_utils import RandomNetworkFailure, random_sleep
 
 nodes = {
     'http://localhost:3000/': 'http://node1:3000/',
@@ -37,9 +40,10 @@ def normalize_url(url: str, inter: bool = False):
         return format
 
 
-def send_resource_to_other_nodes(addresses: list[str], resource: Resource):
+def send_resource_to_other_nodes(addresses: list[str], resource: Resource, source_address: str):
     for address in addresses:
         try:
+            random_sleep()
             result = requests.put(
                 f"{address}/api/duplicate/resource",
                 data=json.dumps({
@@ -50,17 +54,26 @@ def send_resource_to_other_nodes(addresses: list[str], resource: Resource):
                 print('Address', address, 'Status code:',
                       result.status_code, file=sys.stderr)
                 insert_resource_queue(address, resource)
+        except RandomNetworkFailure as e:
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": "Network failure (random intentional)", "statusCode": 418, "node_failed": external_address})
+            insert_resource_queue(address, resource)
         except requests.exceptions.RequestException as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             insert_resource_queue(address, resource)
         except Exception as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             insert_resource_queue(address, resource)
 
 
-def send_command_to_other_nodes(addresses: list[str], resource_id: str, command: Command):
+def send_command_to_other_nodes(addresses: list[str], resource_id: str, command: Command, source_address: str):
     for address in addresses:
         try:
+            random_sleep()
             result = requests.put(
                 f"{address}/api/duplicate/resource/{resource_id}/command",
                 data=json.dumps({
@@ -71,26 +84,45 @@ def send_command_to_other_nodes(addresses: list[str], resource_id: str, command:
                 print('Address', address, 'Status code:',
                       result.status_code, file=sys.stderr)
                 insert_command_queue(address, resource_id, command)
+        except RandomNetworkFailure as e:
+            print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": "Network failure (random intentional)", "statusCode": 418, "node_failed": external_address})
+            insert_command_queue(address, resource_id, command)
         except requests.exceptions.RequestException as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             insert_command_queue(address, resource_id, command)
         except Exception as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.UPDATING, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             insert_command_queue(address, resource_id, command)
 
-def ping(addresses: list[str]):
+def ping(addresses: list[str], source_address: str):
     results = {}
     for address in addresses:
         try:
+            random_sleep()
             result = requests.get(f"{address}/api/ping")
             if result.status_code != 200:
                 print('Address', address, 'Status code:',
                       result.status_code, file=sys.stderr)
                 results[address] = result.status_code == 200
+        except RandomNetworkFailure as e:
+            print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.ONLINE, source_address, {"message": "Network failure (random intentional)", "statusCode": 418, "node_failed": external_address})
+            results[address] = False
         except requests.exceptions.RequestException as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.ONLINE, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             results[address] = False
         except Exception as e:
             print('Address', address, e, file=sys.stderr)
+            external_address = normalize_url(address)
+            update_state(Status.ONLINE, source_address, {"message": e, "statusCode": 500, "node_failed": external_address})
             results[address] = False
     return results
\ No newline at end of file

utils/time_utils.py

+import random
 import time
 
+from consts.numerical_values import (FAILURE_PERCENTAGE, MAX_SLEEP_TIME,
+                                     MIN_SLEEP_TIME)
+
+
+class RandomNetworkFailure(Exception):
+    def __init__(self, message="Random network failure 🙂"):
+        self.message = message
+        super().__init__(self.message)
 
 def current_milli_time():
     return round(time.time() * 1000)
+
+def random_sleep(min_time = MIN_SLEEP_TIME, max_time = MAX_SLEEP_TIME, fail_percentage = FAILURE_PERCENTAGE):
+    if random.random() < fail_percentage:
+        raise RandomNetworkFailure()
+    time.sleep(random.uniform(min_time, max_time))
\ No newline at end of file