Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
M
mock-cheops
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
chephren
mock-cheops
Commits
bf5cd436
Unverified
Commit
bf5cd436
authored
1 year ago
by
REIG Julien
Browse files
Options
Downloads
Patches
Plain Diff
fix ping and parallelize requests
parent
3faaa52a
No related branches found
No related tags found
1 merge request
!18
fix ping and parallelize requests
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
controllers/ping_controller.py
+9
-5
9 additions, 5 deletions
controllers/ping_controller.py
utils/http_utils.py
+87
-75
87 additions, 75 deletions
utils/http_utils.py
utils/thread_value.py
+14
-0
14 additions, 0 deletions
utils/thread_value.py
with
110 additions
and
80 deletions
controllers/ping_controller.py
+
9
−
5
View file @
bf5cd436
...
...
@@ -16,14 +16,18 @@ class PingController(Resource):
if
get_state
()
==
Status
.
OFFLINE
:
return
response_offline
,
403
node_addresses
=
request
.
headers
.
getlist
(
"
addresses
"
)
if
(
node_addresses
is
None
or
len
(
node_addresses
)
==
0
):
return
{
"
message
"
:
"
No addresses provided.
"
},
400
node_addresses
=
request
.
headers
.
get
(
"
addresses
"
,
None
)
print
(
'
node_addresses
'
,
node_addresses
)
if
(
node_addresses
is
None
):
return
jsonify
({
"
reachable
"
:
True
,
"
results
"
:
{
normalize_url
(
request
.
host_url
):
True
}
})
node_addresses
=
node_addresses
.
split
(
"
,
"
)
node_addresses
=
list
(
map
(
normalize_url
,
node_addresses
))
results
=
ping
(
node_addresses
)
if
(
normalize_url
(
request
.
host_url
)
in
node_addresses
):
results
[
normalize_url
(
request
.
host_url
)]
=
True
node_addresses
.
remove
(
normalize_url
(
request
.
host_url
))
results
=
ping
(
node_addresses
,
normalize_url
(
request
.
host_url
))
results
[
normalize_url
(
request
.
host_url
)]
=
True
return
jsonify
({
"
reachable
"
:
True
,
"
results
"
:
results
})
...
...
This diff is collapsed.
Click to expand it.
utils/http_utils.py
+
87
−
75
View file @
bf5cd436
import
json
import
sys
import
threading
from
urllib.parse
import
urlparse
import
requests
...
...
@@ -9,6 +10,7 @@ from models.command import Command
from
models.resource
import
Resource
from
utils.database_utils
import
(
insert_command_queue
,
insert_resource_queue
,
update_state
)
from
utils.thread_value
import
Thread_value
from
utils.time_utils
import
RandomNetworkFailure
,
random_sleep
nodes
=
{
...
...
@@ -38,91 +40,101 @@ def normalize_url(url: str, inter: bool = False):
return
url
else
:
return
format
def
send_data_in_thread
(
address
:
str
,
data
:
str
,
source_address
:
str
,
on_error_callback
:
callable
=
None
):
try
:
random_sleep
()
result
=
requests
.
put
(
address
,
data
=
data
,
headers
=
{
"
Content-Type
"
:
"
application/json
"
})
if
result
.
status_code
!=
201
:
print
(
'
Address
'
,
address
,
'
Status code:
'
,
result
.
status_code
,
file
=
sys
.
stderr
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
"
Status code not 201
"
,
"
statusCode
"
:
result
.
status_code
,
"
node_failed
"
:
address
})
if
on_error_callback
:
on_error_callback
()
except
RandomNetworkFailure
as
e
:
print
(
'
Address (intentional network fail)
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
"
Network failure (random intentional)
"
,
"
statusCode
"
:
418
,
"
node_failed
"
:
external_address
})
if
on_error_callback
:
on_error_callback
()
except
requests
.
exceptions
.
RequestException
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
if
on_error_callback
:
on_error_callback
()
except
Exception
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
if
on_error_callback
:
on_error_callback
()
def
send_resource_to_other_nodes
(
addresses
:
list
[
str
],
resource
:
Resource
,
source_address
:
str
):
'''
Send resource to other nodes using threads
'''
all_threads
=
[]
for
address
in
addresses
:
try
:
random_sleep
()
result
=
requests
.
put
(
f
"
{
address
}
/api/duplicate/resource
"
,
data
=
json
.
dumps
({
"
resource
"
:
resource
.
to_full_dict
(
include_presence
=
True
),
}),
headers
=
{
"
Content-Type
"
:
"
application/json
"
})
if
result
.
status_code
!=
201
:
print
(
'
Address
'
,
address
,
'
Status code:
'
,
result
.
status_code
,
file
=
sys
.
stderr
)
insert_resource_queue
(
address
,
resource
)
except
RandomNetworkFailure
as
e
:
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
"
Network failure (random intentional)
"
,
"
statusCode
"
:
418
,
"
node_failed
"
:
external_address
})
insert_resource_queue
(
address
,
resource
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
insert_resource_queue
(
address
,
resource
)
except
Exception
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
insert_resource_queue
(
address
,
resource
)
data
=
json
.
dumps
({
"
resource
"
:
resource
.
to_full_dict
(
include_presence
=
True
),})
on_error_callback
=
lambda
:
insert_resource_queue
(
address
,
resource
)
th
=
threading
.
Thread
(
target
=
send_data_in_thread
,
args
=
(
f
"
{
address
}
/api/duplicate/resource
"
,
data
,
source_address
,
on_error_callback
))
all_threads
.
append
(
th
)
th
.
start
()
for
th
in
all_threads
:
th
.
join
()
def
send_command_to_other_nodes
(
addresses
:
list
[
str
],
resource_id
:
str
,
command
:
Command
,
source_address
:
str
):
'''
Send command to other nodes using threads
'''
all_threads
=
[]
for
address
in
addresses
:
try
:
random_sleep
()
result
=
requests
.
put
(
f
"
{
address
}
/api/duplicate/resource/
{
resource_id
}
/command
"
,
data
=
json
.
dumps
({
"
command
"
:
command
.
to_dict
(),
}),
headers
=
{
"
Content-Type
"
:
"
application/json
"
})
if
result
.
status_code
!=
201
:
print
(
'
Address
'
,
address
,
'
Status code:
'
,
result
.
status_code
,
file
=
sys
.
stderr
)
insert_command_queue
(
address
,
resource_id
,
command
)
except
RandomNetworkFailure
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
"
Network failure (random intentional)
"
,
"
statusCode
"
:
418
,
"
node_failed
"
:
external_address
})
insert_command_queue
(
address
,
resource_id
,
command
)
except
requests
.
exceptions
.
RequestException
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
insert_command_queue
(
address
,
resource_id
,
command
)
except
Exception
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
UPDATING
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
insert_command_queue
(
address
,
resource_id
,
command
)
data
=
json
.
dumps
({
"
command
"
:
command
.
to_dict
(),})
on_error_callback
=
lambda
:
insert_command_queue
(
address
,
resource_id
,
command
)
th
=
threading
.
Thread
(
target
=
send_data_in_thread
,
args
=
(
f
"
{
address
}
/api/duplicate/resource/
{
resource_id
}
/command
"
,
data
,
source_address
,
on_error_callback
))
all_threads
.
append
(
th
)
th
.
start
()
for
th
in
all_threads
:
th
.
join
()
def
ping_in_thread
(
address
:
str
,
source_address
:
str
):
try
:
random_sleep
()
addr
=
f
"
{
normalize_url
(
address
,
True
)
}
/api/ping
"
result
=
requests
.
get
(
addr
)
if
result
.
status_code
!=
200
:
print
(
'
Address
'
,
addr
,
'
Status code:
'
,
result
.
status_code
,
result
.
reason
,
file
=
sys
.
stderr
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
"
Status code not 200
"
,
"
statusCode
"
:
result
.
status_code
,
"
node_failed
"
:
address
})
return
address
,
result
.
status_code
==
200
except
RandomNetworkFailure
as
e
:
print
(
'
RandomNetworkFailure: Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
"
Network failure (random intentional)
"
,
"
statusCode
"
:
418
,
"
node_failed
"
:
address
})
return
address
,
False
except
requests
.
exceptions
.
RequestException
as
e
:
print
(
'
RequestException: Address
'
,
addr
,
e
,
file
=
sys
.
stderr
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
address
})
return
address
,
False
except
Exception
as
e
:
print
(
'
Exception: Address
'
,
addr
,
e
,
file
=
sys
.
stderr
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
address
})
return
address
,
False
def
ping
(
addresses
:
list
[
str
],
source_address
:
str
):
all_threads
=
[]
results
=
{}
print
(
'
pinging
'
,
addresses
,
flush
=
True
)
for
address
in
addresses
:
try
:
random_sleep
()
result
=
requests
.
get
(
f
"
{
address
}
/api/ping
"
)
if
result
.
status_code
!=
200
:
print
(
'
Address
'
,
address
,
'
Status code:
'
,
result
.
status_code
,
file
=
sys
.
stderr
)
results
[
address
]
=
result
.
status_code
==
200
except
RandomNetworkFailure
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
"
Network failure (random intentional)
"
,
"
statusCode
"
:
418
,
"
node_failed
"
:
external_address
})
results
[
address
]
=
False
except
requests
.
exceptions
.
RequestException
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
results
[
address
]
=
False
except
Exception
as
e
:
print
(
'
Address
'
,
address
,
e
,
file
=
sys
.
stderr
)
external_address
=
normalize_url
(
address
)
update_state
(
Status
.
ONLINE
,
source_address
,
{
"
message
"
:
e
,
"
statusCode
"
:
500
,
"
node_failed
"
:
external_address
})
results
[
address
]
=
False
print
(
'
ping
'
,
address
,
flush
=
True
)
th
=
Thread_value
(
target
=
ping_in_thread
,
args
=
(
address
,
source_address
))
all_threads
.
append
(
th
)
th
.
start
()
for
th
in
all_threads
:
address
,
value
=
th
.
join
()
results
[
address
]
=
value
return
results
\ No newline at end of file
This diff is collapsed.
Click to expand it.
utils/thread_value.py
0 → 100644
+
14
−
0
View file @
bf5cd436
from
threading
import
Thread
class
Thread_value
(
Thread
):
def
__init__
(
self
,
target
,
args
):
super
().
__init__
(
target
=
target
,
args
=
args
)
self
.
_return
=
None
def
run
(
self
):
self
.
_return
=
self
.
_target
(
*
self
.
_args
)
def
join
(
self
):
super
().
join
()
return
self
.
_return
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment