VladislavD
/
monitor
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects Releases 1 Wiki Activity

refactor + service checking bug with repeating twice

pull/9/head
prplV 2024-09-10 17:43:19 +03:00
parent deebb77ffb
commit 8c71453f29
9 changed files with 290 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/config.rs
View File

@ -1,10 +1,12 @@
use std::fs;
use crate::structs::*;
use log::{error, info, warn};
use redis::{Client, Commands, Connection};
use redis::{Client, Commands, Connection, RedisResult};
use std::fs;
use tokio::time::Duration;
static CONFIG_PATH : &'static str = "settings.json";
static CONFIG_PATH: &str = "settings.json";
type Res = RedisResult<Vec<Vec<(String, Vec<(String, String)>)>>>;
// 4ever sync
fn load_processes(json_filename: &str) -> Option<Processes> {
if let Ok(res) = fs::read_to_string(json_filename) {
@ -18,39 +20,39 @@ fn load_processes(json_filename: &str) -> Option<Processes>{
pub fn get_actual_config() -> Option<Processes> {
// * if no conf -> loop and +inf getting conf from redis server
// let mut local = load_processes(&CONFIG_PATH);
match load_processes(&CONFIG_PATH) {
match load_processes(CONFIG_PATH) {
Some(local_conf) => {
if let Some(remote_conf) = once_get_remote_configuration("redis://localhost") {
match config_comparing(&local_conf, &remote_conf) {
return match config_comparing(&local_conf, &remote_conf) {
ConfigActuality::Local => {
info!("Local config is actual");
return Some(local_conf);
},
Some(local_conf)
}
ConfigActuality::Remote => {
info!("Pulled config is more actual. Saving changes!");
if save_new_config(&remote_conf, &CONFIG_PATH).is_err() {
if save_new_config(&remote_conf, CONFIG_PATH).is_err() {
error!("Saving changes process failed due to unexpected error...")
}
return Some(remote_conf);
},
Some(remote_conf)
}
};
}
Some(local_conf)
},
}
None => {
// ? ? OUTSTANDING CONSTRUCTION ?
let mut conn = get_connection_watcher(&open_watcher("redis://localhost"));
get_stream_info_watcher(&mut conn);
let remote_config = invalid_config_watcher(&mut conn);
let _ = save_new_config(&remote_config, &CONFIG_PATH);
let _ = save_new_config(&remote_config, CONFIG_PATH);
Some(remote_config)
},
}
}
}
// ! once iter exec
// ! only for situation when local isn't None (no need to fck redis server)
fn once_get_remote_configuration(serv_info: &str) -> Option<Processes> {
match redis::Client::open(serv_info) {
match Client::open(serv_info) {
Ok(client) => {
match client.get_connection() {
Ok(mut conn) => {
@ -59,7 +61,7 @@ fn once_get_remote_configuration(serv_info: &str) -> Option<Processes> {
warn!("No configuration in DB yet");
None
} else {
let conf: redis::RedisResult<Vec<Vec<(String, Vec<(String, String)>)>>> = conn.xrevrange_count("config_stream", "+", "-", 1);
let conf: Res = conn.xrevrange_count("config_stream", "+", "-", 1);
let config: &Vec<(String, Vec<(String, String)>)>;
if conf.is_ok() {
@ -67,7 +69,9 @@ fn once_get_remote_configuration(serv_info: &str) -> Option<Processes> {
let conf = conf.unwrap();
config = &conf[0];
if config.is_empty() {
error!("Empty config was pulled. Check stream and configs state!");
error!(
"Empty config was pulled. Check stream and configs state!"
);
return None;
}
match parse_extern_config(&config[0].1[0].1) {
@ -75,7 +79,7 @@ fn once_get_remote_configuration(serv_info: &str) -> Option<Processes> {
None => {
error!("Invalid configuration was pulled (PARSING_ERROR). Check configs state!");
None
},
}
}
} else {
error!("Configuration pulling from Redis stream failed. Check stream state!");
@ -86,29 +90,29 @@ fn once_get_remote_configuration(serv_info: &str) -> Option<Processes> {
error!("Cannot find config_stream. Check Redis-stream accessibility!");
None
}
},
}
Err(_) => {
error!("Redis connection attempt is failed. Check Redis configuration!");
None
},
}
},
}
}
Err(_) => {
error!("Redis-Client opening attempt is failed. Check network configuration!");
None
},
}
}
}
// ! watchers
fn open_watcher(serv_info: &str) -> redis::Client {
fn open_watcher(serv_info: &str) -> Client {
loop {
match redis::Client::open(serv_info) {
match Client::open(serv_info) {
Ok(redis) => {
info!("Successfully opened Redis-Client");
return redis
},
return redis;
}
Err(_) => {
error!("Redis-Client opening attempt is failed. Check network configuration! Retrying...");
std::thread::sleep(Duration::from_secs(4));
@ -121,11 +125,13 @@ fn get_connection_watcher(client: &Client) -> Connection {
loop {
match client.get_connection() {
Ok(conn) => {
info!("Succesfully got Redis connection object");
info!("Successfully got Redis connection object");
return conn;
},
}
Err(_) => {
error!("Redis connection attempt is failed. Check Redis configuration! Retrying...");
error!(
"Redis connection attempt is failed. Check Redis configuration! Retrying..."
);
std::thread::sleep(Duration::from_secs(4));
}
}
@ -144,9 +150,8 @@ fn get_stream_info_watcher(conn: &mut Connection) {
}
}
fn invalid_config_watcher(conn: &mut Connection) -> Processes {
// let res: redis::RedisResult<Vec<Vec<(String, Vec<(String, String)>)>>>;
loop {
let res: redis::RedisResult<Vec<Vec<(String, Vec<(String, String)>)>>> = conn.xrevrange_count("config_stream", "+", "-", 1);
let res: Res = conn.xrevrange_count("config_stream", "+", "-", 1);
if res.is_ok() {
let config = &res.unwrap()[0];
if !config.is_empty() {
@ -167,14 +172,13 @@ fn config_comparing(local: &Processes, remote: &Processes) -> ConfigActuality {
let remote_date: u64 = remote.date_of_creation.parse().unwrap();
match local_date.cmp(&remote_date) {
std::cmp::Ordering::Equal |
std::cmp::Ordering::Greater => return ConfigActuality::Local,
std::cmp::Ordering::Less => return ConfigActuality::Remote,
std::cmp::Ordering::Equal | std::cmp::Ordering::Greater => ConfigActuality::Local,
std::cmp::Ordering::Less => ConfigActuality::Remote,
}
}
// ! TEMPORARLY DEPRICATED !
// fn native_date_from_milis(mls: &str) -> Option<chrono::DateTime<Utc>> {
// ! TEMPORARILY DEPRECATED !
// fn native_date_from_millis(mls: &str) -> Option<chrono::DateTime<Utc>> {
// match mls.parse::<i64>(){
// Ok(val) => return chrono::DateTime::from_timestamp_millis(val),
// Err(_) => return None,
@ -183,21 +187,17 @@ fn config_comparing(local: &Processes, remote: &Processes) -> ConfigActuality {
fn save_new_config(config: &Processes, config_file: &str) -> Result<(), CustomError> {
match serde_json::to_string_pretty(&config) {
Ok(st) => {
match fs::write(config_file, st) {
Ok(_) => return Ok(()),
Err(_) => return Err(CustomError::Fatal),
}
Ok(st) => match fs::write(config_file, st) {
Ok(_) => Ok(()),
Err(_) => Err(CustomError::Fatal),
},
Err(_) => return Err(CustomError::Fatal),
Err(_) => Err(CustomError::Fatal),
}
}
fn parse_extern_config(json_string: &str) -> Option<Processes> {
let des = serde_json::from_str::<Processes>(json_string);
if des.is_err() {
return None;
} else {
return Some(des.unwrap());
if let Ok(des) = serde_json::from_str::<Processes>(json_string){
return Some(des);
}
None
}

52
src/files.rs
View File

@ -1,12 +1,12 @@
use crate::structs::{Files, CustomError};
use crate::prcs::{is_active, is_frozen};
use crate::structs::{CustomError, Files};
use inotify::{EventMask, Inotify, WatchMask};
use log::error;
use tokio::sync::mpsc;
use crate::prcs::{is_frozen, is_active};
use tokio::time::Duration;
use std::sync::Arc;
use std::path::Path;
use std::borrow::BorrowMut;
use std::path::Path;
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::time::Duration;
pub async fn create_watcher(filename: &str, path: &str) -> Result<Inotify, std::io::Error> {
let src = format!("{}{}", path, filename);
@ -14,24 +14,17 @@ pub async fn create_watcher(filename: &str, path: &str) -> Result<Inotify, std::
error!("{}", format!("Cannot create watcher for {}", &src));
std::process::exit(101);
});
_ = inotify
.watches()
.add(
&src,
WatchMask::ALL_EVENTS
);
_ = inotify.watches().add(&src, WatchMask::ALL_EVENTS);
Ok(inotify)
}
pub async fn file_handler
(
pub async fn file_handler(
name: &str,
files: &Vec<Files>,
files: &[Files],
tx: Arc<mpsc::Sender<u8>>,
watchers: Arc<tokio::sync::Mutex<Vec<Inotify>>>
) -> Result<(), CustomError>
{
watchers: Arc<tokio::sync::Mutex<Vec<Inotify>>>,
) -> Result<(), CustomError> {
for (i, file) in files.iter().enumerate() {
// let src = format!("{}{}", file.src, file.filename);
if check_file(&file.filename, &file.src).await.is_err() {
@ -41,24 +34,24 @@ pub async fn file_handler
match file.triggers.on_delete.as_str() {
"stay" => {
continue;
},
}
"stop" => {
if is_active(name).await {
tx.send(1).await.unwrap();
}
return Err(CustomError::Fatal);
},
}
"hold" => {
if is_active(name).await {
tx.send(2).await.unwrap();
return Err(CustomError::Fatal);
}
},
}
_ => {
tokio::time::sleep(Duration::from_millis(50)).await;
tx.send(101).await.unwrap();
return Err(CustomError::Fatal);
},
}
}
} else if is_active(name).await && !is_frozen(name).await {
let watchers = watchers.clone();
@ -69,9 +62,9 @@ pub async fn file_handler
let events = notify.read_events(&mut buffer);
// println!("{:?}", events);
if events.is_ok() {
let events: Vec<EventMask> = events.unwrap()
.into_iter()
.map(|mask| {mask.mask})
let events: Vec<EventMask> = events
.unwrap()
.map(|mask| mask.mask)
.filter(|mask| {
*mask == EventMask::MODIFY || *mask == EventMask::DELETE_SELF
})
@ -88,16 +81,16 @@ pub async fn file_handler
match file.triggers.on_change.as_str() {
"stop" => {
let _ = tx.send(7).await;
},
}
"restart" => {
let _ = tx.send(8).await;
},
}
"stay" => {
let _ = tx.send(9).await;
},
}
_ => {
let _ = tx.send(101).await;
},
}
}
}
}
@ -108,7 +101,6 @@ pub async fn file_handler
Ok(())
}
pub async fn check_file(filename: &str, path: &str) -> Result<(), CustomError> {
let arc_name = Arc::new(filename.to_string());
let arc_path = Arc::new(path.to_string());

7
src/logger.rs
View File

@ -2,8 +2,8 @@
// use std::process::Command;
use chrono::Local;
use env_logger::Builder;
use std::io::Write;
use log::LevelFilter;
use std::io::Write;
// use tokio::fs::File;
// use tokio::io::BufWriter;
use crate::utils::get_container_id;
@ -24,11 +24,11 @@ pub fn setup_logger() -> Result<(), crate::structs::CustomError> {
// std::process::exit(1);
// }));
// building logger with current output format
Builder::new()
.format(move |buf, record| {
writeln!(buf,
writeln!(
buf,
"Container-{}| {} [{}] - {}",
get_container_id().unwrap_or("||NODE|".to_string()),
Local::now().format("%d-%m-%Y %H:%M:%S"),
@ -42,6 +42,5 @@ pub fn setup_logger() -> Result<(), crate::structs::CustomError> {
// .target(env_logger::Target::Pipe(log_target))
.init();
Ok(())
}

33
src/main.rs
View File

@ -1,21 +1,20 @@
mod structs;
mod config;
mod files;
mod prcs;
mod utils;
mod services;
mod logger;
mod prcs;
mod services;
mod signals;
mod structs;
mod utils;
use tokio::sync::mpsc;
use std::sync::Arc;
use log::{error, info};
use structs::*;
use config::*;
use utils::*;
use log::{error, info};
use logger::setup_logger;
use signals::set_valid_destructor;
use std::sync::Arc;
use structs::*;
use tokio::sync::mpsc;
use utils::*;
#[tokio::main(flavor = "multi_thread")]
async fn main() {
@ -30,7 +29,10 @@ async fn main() {
std::process::exit(101);
});
log::info!("Current runner configuration: {}", &processes.date_of_creation);
log::info!(
"Current runner configuration: {}",
&processes.date_of_creation
);
log::info!("Runner is ready. Initializing...");
if processes.processes.len() == 0 {
@ -42,7 +44,8 @@ async fn main() {
let mut senders: Vec<Arc<mpsc::Sender<u8>>> = vec![];
for proc in processes.processes.iter() {
log::info!("Process '{}' on stage: {}. Depends on {} file(s), {} service(s)",
log::info!(
"Process '{}' on stage: {}. Depends on {} file(s), {} service(s)",
proc.name,
proc.path,
proc.dependencies.files.len(),
@ -63,14 +66,12 @@ async fn main() {
handler.push(event);
}
// destructor addition
handler.push(
tokio::spawn(async move {
handler.push(tokio::spawn(async move {
if let Err(_) = set_valid_destructor(Arc::new(senders)).await {
error!("Linux signals handler creation failed. Returning...");
return;
}
})
);
}));
for i in handler {
i.await.unwrap();
}

18
src/prcs.rs
View File

@ -1,8 +1,8 @@
use std::sync::Arc;
use std::process::{ Command, Output };
use log::{error, warn};
use tokio::time::Duration;
use crate::structs::CustomError;
use log::{error, warn};
use std::process::{Command, Output};
use std::sync::Arc;
use tokio::time::Duration;
pub async fn get_pid(name: &str) -> Output {
let name = Arc::new(name.to_string());
@ -43,7 +43,7 @@ pub async fn is_frozen(name: &str) -> bool {
let pid = pid.trim();
let arc_pid = Arc::new(pid.to_string());
if pid.is_empty() {
return false;
false
} else {
tokio::task::spawn_blocking(move || {
let cmd = Command::new("ps")
@ -90,7 +90,7 @@ pub async fn unfreeze_process(name: &str) {
pub async fn restart_process(name: &str, path: &str) -> Result<(), CustomError> {
terminate_process(name).await;
tokio::time::sleep(Duration::from_millis(100)).await;
return start_process(name, path).await;
start_process(name, path).await
}
pub async fn start_process(name: &str, path: &str) -> Result<(), CustomError> {
@ -102,9 +102,7 @@ pub async fn start_process(name: &str, path: &str) -> Result<(), CustomError> {
Ok(_) => {
warn!("Process {} is running now!", name);
Ok(())
},
Err(_) => {
return Err(CustomError::Fatal)
},
}
Err(_) => Err(CustomError::Fatal),
}
}

73
src/services.rs
View File

@ -1,44 +1,49 @@
use std::sync::Arc;
use tokio::time::{ Duration, Instant };
use tokio::sync::mpsc;
use crate::structs::{Services, CustomError};
use crate::prcs::{is_active, is_frozen};
use crate::structs::{CustomError, Services};
use log::{error, warn};
use std::net::{TcpStream, ToSocketAddrs};
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::time::{Duration, Instant};
pub async fn service_handler(name: &str, services: &Vec<Services>, tx: Arc<mpsc::Sender<u8>>) -> Result<(), CustomError> {
pub async fn service_handler(
name: &str,
services: &Vec<Services>,
tx: Arc<mpsc::Sender<u8>>,
) -> Result<(), CustomError> {
// println!("service daemon on {}", name);
for serv in services {
if check_service(&serv.hostname, &serv.port).await.is_err() {
if !is_active(name).await || is_frozen(name).await {
return Err(CustomError::Fatal);
}
error!("Service {}:{} is unreachable for process {}", &serv.hostname, &serv.port, &name);
error!(
"Service {}:{} is unreachable for process {}",
&serv.hostname, &serv.port, &name
);
match serv.triggers.on_lost.as_str() {
"stay" => {
},
"stay" => {}
"stop" => {
if looped_service_connecting(name, serv).await.is_err() {
tx.send(5).await.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
tokio::time::sleep(Duration::from_millis(400)).await;
return Err(CustomError::Fatal);
}
},
}
"hold" => {
if is_frozen(name).await {
return Err(CustomError::Fatal);
}
if looped_service_connecting(name, serv).await.is_err() {
tx.send(6).await.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
tokio::time::sleep(Duration::from_millis(400)).await;
return Err(CustomError::Fatal);
}
},
}
_ => {
tx.send(101).await.unwrap();
return Err(CustomError::Fatal);
},
}
}
}
}
@ -47,23 +52,26 @@ pub async fn service_handler(name: &str, services: &Vec<Services>, tx: Arc<mpsc:
Ok(())
}
async fn looped_service_connecting(
name: &str,
serv: &Services
) -> Result<(), CustomError>
{
async fn looped_service_connecting(name: &str, serv: &Services) -> Result<(), CustomError> {
if serv.triggers.wait == 0 {
loop {
tokio::time::sleep(Duration::from_secs(serv.triggers.delay.into())).await;
warn!("Attempting to connect from {} process to {}:{}",&name, &serv.hostname, &serv.port );
warn!(
"Attempting to connect from {} process to {}:{}",
&name, &serv.hostname, &serv.port
);
match check_service(&serv.hostname, &serv.port).await {
Ok(_) => {
log::info!("Successfully connected to {} from {} process!", &serv.hostname, &name);
log::info!(
"Successfully connected to {} from {} process!",
&serv.hostname,
&name
);
break;
},
}
Err(_) => {
continue;
},
}
}
}
Ok(())
@ -71,15 +79,22 @@ async fn looped_service_connecting(
let start = Instant::now();
while start.elapsed().as_secs() < serv.triggers.wait.into() {
tokio::time::sleep(Duration::from_secs(serv.triggers.delay.into())).await;
warn!("Attempting to connect from {} process to {}:{}",&name, &serv.hostname, &serv.port );
warn!(
"Attempting to connect from {} process to {}:{}",
&name, &serv.hostname, &serv.port
);
match check_service(&serv.hostname, &serv.port).await {
Ok(_) => {
log::info!("Successfully connected to {} from {} process!", &serv.hostname, &name);
log::info!(
"Successfully connected to {} from {} process!",
&serv.hostname,
&name
);
return Ok(());
},
}
Err(_) => {
continue;
},
}
}
}
Err(CustomError::Fatal)
@ -93,12 +108,12 @@ async fn check_service(hostname: &str, port: &u32) -> Result<(), CustomError> {
match addr.to_socket_addrs() {
Ok(mut addrs) => {
if let Some(_) = addrs.find(|a| TcpStream::connect_timeout(a, std::time::Duration::new(1, 0)).is_ok()) {
if addrs.any(|a| TcpStream::connect_timeout(&a, Duration::new(1, 0)).is_ok()) {
Ok(())
} else {
Err(CustomError::Fatal)
}
},
}
Err(_) => Err(CustomError::Fatal),
}
}

4
src/signals.rs
View File

@ -1,11 +1,11 @@
use crate::structs::CustomError;
use std::sync::Arc;
use tokio::io;
use tokio::sync::mpsc;
use tokio::{
select,
signal::unix::{signal, Signal, SignalKind},
};
use tokio::sync::mpsc;
type SendersVec = Arc<Vec<Arc<mpsc::Sender<u8>>>>;
@ -67,7 +67,7 @@ impl SigPostProcessing for Sig {
async fn post_processing(&mut self) -> io::Result<()> {
// manipulations ...
if let Some(_) = self.signal.recv().await {
log::info!("Got {}", self.sig_type);
log::info!("Got {} signal", self.sig_type);
for prc in self.senders.clone().iter() {
prc.send(111).await.unwrap();
}

64
src/utils.rs
View File

@ -1,25 +1,21 @@
use std::sync::Arc;
use crate::structs::TrackingProcess;
use tokio::sync::mpsc;
use inotify::Inotify;
use std::process::Command;
use crate::files::create_watcher;
use log::{error, warn};
use crate::prcs::{
is_active,
is_frozen,
terminate_process,
restart_process,
freeze_process,
unfreeze_process,
start_process
};
use tokio::time::Duration;
use tokio::join;
use crate::files::file_handler;
use crate::prcs::{
freeze_process, is_active, is_frozen, restart_process, start_process, terminate_process,
unfreeze_process,
};
use crate::services::service_handler;
use crate::structs::TrackingProcess;
use inotify::Inotify;
use log::{error, warn};
use std::process::Command;
use std::sync::Arc;
use tokio::join;
use tokio::sync::mpsc;
use tokio::time::Duration;
static GET_ID_CMD : &'static str = r"cat /proc/self/mountinfo | grep '/docker/containers/' | head -1 | awk -F '/' '{print \$6}'";
static GET_ID_CMD: &str =
r"cat /proc/self/mountinfo | grep '/docker/containers/' | head -1 | awk -F '/' '{print \$6}'";
/// # async func to run 3 main daemons (now it's more like tree-form than classical 0.1.0 form )
/// > hint : give mpsc with capacity 1 to jump over potential errors during running process
@ -27,15 +23,15 @@ static GET_ID_CMD : &'static str = r"cat /proc/self/mountinfo | grep '/docker/co
pub async fn run_daemons(
proc: Arc<TrackingProcess>,
tx: Arc<mpsc::Sender<u8>>,
rx: &mut mpsc::Receiver<u8>
)
{
rx: &mut mpsc::Receiver<u8>,
) {
// creating watchers + ---buffers---
let mut watchers: Vec<Inotify> = vec![];
for file in proc.dependencies.files.clone().into_iter() {
watchers.push(create_watcher(&file.filename, &file.src).await.unwrap());
}
let watchers_clone: Arc<tokio::sync::Mutex<Vec<Inotify>>> = Arc::new(tokio::sync::Mutex::new(watchers));
let watchers_clone: Arc<tokio::sync::Mutex<Vec<Inotify>>> =
Arc::new(tokio::sync::Mutex::new(watchers));
loop {
let run_hand = running_handler(proc.clone(), tx.clone(), watchers_clone.clone());
@ -146,15 +142,18 @@ pub async fn run_daemons(
}
}
// check process status daemon
pub async fn running_handler
(
pub async fn running_handler(
prc: Arc<TrackingProcess>,
tx: Arc<mpsc::Sender<u8>>,
watchers: Arc<tokio::sync::Mutex<Vec<Inotify>>>
)
{
watchers: Arc<tokio::sync::Mutex<Vec<Inotify>>>,
) {
// services and files check (once)
let files_check = file_handler(&prc.name, &prc.dependencies.files, tx.clone(), watchers.clone());
let files_check = file_handler(
&prc.name,
&prc.dependencies.files,
tx.clone(),
watchers.clone(),
);
let services_check = service_handler(&prc.name, &prc.dependencies.services, tx.clone());
let res = join!(files_check, services_check);
@ -174,15 +173,10 @@ pub async fn running_handler
tokio::task::yield_now().await;
}
// todo: cmd across cat /proc/self/mountinfo | grep "/docker/containers/" | head -1 | awk -F '/' '{print $5}'
pub fn get_container_id() -> Option<String> {
match Command::new("sh -c").arg(GET_ID_CMD).output() {
Ok(output) => {
Some(String::from_utf8_lossy(&output.stdout).to_string())
},
Err(_) => {
None
},
Ok(output) => Some(String::from_utf8_lossy(&output.stdout).to_string()),
Err(_) => None,
}
}