Commit 8722ace6 authored by Johannes Barthel's avatar Johannes Barthel
Browse files

use mongo instead of bbb api, clean up etherpads

parent b81195de
BBB_API_URL=https://instance.your-bbb-server.tld/bigbluebutton/
BBB_API_SECRET=redacted
This diff is collapsed.
[package]
name = "cleanup-meeting-data"
version = "0.1.1"
authors = ["Johannes <johannes.barthel@student.kit.edu>"]
version = "0.2.0"
authors = ["Johannes <johannes@senfcall.de>"]
edition = "2018"
[dependencies]
dotenv = "0.15.0"
sha-1 = "0.8.2"
url = "2.1.1"
regex = "1.3.7"
reqwest = { version = "0.10.4", features = ["blocking"] }
[package.metadata.deb]
maintainer = "Johannes <johannes.barthel@student.kit.edu>"
copyright = "2020 Johannes <johannes.barthel@student.kit.edu>"
maintainer = "Johannes <johannes@senfcall.de>"
copyright = "2021 Johannes <johannes@senfcall.de>"
license-file = ["LICENSE", "0"]
depends = "$auto, systemd"
extended-description = """\
......@@ -26,3 +19,10 @@ assets = [
["target/release/cleanup-meeting-data", "usr/bin/", "755"]
]
maintainer-scripts = "debian"
[dependencies]
regex = "1.3.7"
serde = { version = "1.0", features = ["derive"] }
serde_derive = "1.0"
reqwest = { version = "0.11", features = ["blocking", "json"] }
serde_json = "1.0"
use std::env;
use dotenv::dotenv;
use url::form_urlencoded::Serializer;
use sha1::{Sha1, Digest};
use std::fs;
use serde::Deserialize;
use std::collections::HashSet;
use std::fs;
fn sha1(input : &str) -> String {
let mut sha = Sha1::new();
sha.input(input);
format!("{:x}", sha.result())
#[derive(Deserialize)]
struct PadList {
#[serde(rename="padIDs")]
pad_ids: Vec<String>,
}
fn checksum (method : &str, query : &str) -> String {
let bbb_api_secret : String = env::var("BBB_API_SECRET").unwrap();
sha1(&format!("{}{}{}", method, query, &bbb_api_secret))
#[derive(Deserialize)]
struct EtherpadApiQuery {
data: PadList,
}
fn build_api_url(endpoint : &str, params : &[(&str, &str)]) -> String {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct EtherpadApiVersion {
current_version: String,
}
let bbb_api_url : String = env::var("BBB_API_URL").unwrap();
let api = if bbb_api_url.ends_with('/') { "api/" } else { "/api/" };
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct MongoRunningMeeting {
meeting_id : String,
}
let mut ser = Serializer::new(String::new());
for (k, v) in params.iter() {
ser.append_pair(k, v);
}
let params : String = ser.finish();
let hash = checksum(endpoint, &params);
let url_base = format!("{}{}{}?{}", bbb_api_url, api, endpoint, params);
format!("{}&checksum={}", url_base, hash)
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct MongoCaption {
meeting_id : String,
pad_id : String
}
fn main() {
dotenv().unwrap();
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct MongoNote {
meeting_id : String,
note_id : String
}
fn get_meetings_on_disk() -> HashSet<String> {
let re = regex::Regex::new(r"^[0-9a-f]{40}-\d+$").unwrap();
let meetings_on_disk : HashSet<String> = fs::read_dir("/var/bigbluebutton/").unwrap().filter_map(|e| {
let as_string = e.unwrap().file_name().to_str().unwrap().to_owned();
if re.is_match(&as_string) {
Some(as_string)
} else {
None
}
}).collect();
fs::read_dir("/var/bigbluebutton/")
.unwrap()
.filter_map(|e| {
let as_string = e.unwrap().file_name().to_str().unwrap().to_owned();
if re.is_match(&as_string) {
Some(as_string)
} else {
None
}
})
.collect()
}
fn get_stored_notes(etherpad_api_version: &str, etherpad_api_key: &str) -> Vec<String> {
let api_call = format!(
"http://localhost:9001/api/{}/listAllPads?apikey={}",
&etherpad_api_version, &etherpad_api_key
);
dbg!(&api_call);
reqwest::blocking::get(&api_call)
.unwrap()
.json::<EtherpadApiQuery>()
.unwrap()
.data
.pad_ids
}
fn get_mongo_running_meetings() -> HashSet<String> {
let stdout = String::from_utf8(std::process::Command::new("mongo").args(&[
"--quiet", "--eval", r#"db.meetings.find({},{"meetingId": true, "_id": false}).toArray()"#,
"127.0.1.1/meteor"
])
.output().unwrap().stdout).unwrap();
let running_meetings : Vec<MongoRunningMeeting> = serde_json::from_str(&stdout).unwrap();
running_meetings.into_iter().map(|running_meeting| running_meeting.meeting_id).collect()
}
fn get_mongo_subtitles_by_meeting_id() -> HashSet<(String, String)> {
let stdout = String::from_utf8(std::process::Command::new("mongo").args(&[
"--quiet", "--eval", r#"db.captions.find({},{"meetingId": true, "padId": true, "_id": false}).toArray()"#,
"127.0.1.1/meteor"
])
.output().unwrap().stdout).unwrap();
let subtitles : Vec<MongoCaption> = serde_json::from_str(&stdout).unwrap();
subtitles.into_iter().map(|subtitle| (subtitle.meeting_id, subtitle.pad_id)).collect()
}
fn get_mongo_notes_by_meeting_id() -> HashSet<(String, String)> {
let stdout = String::from_utf8(std::process::Command::new("mongo").args(&[
"--quiet", "--eval", r#"db.note.find({},{"meetingId": true, "noteId": true, "_id": false}).toArray()"#,
"127.0.1.1/meteor"
])
.output().unwrap().stdout).unwrap();
let subtitles : Vec<MongoNote> = serde_json::from_str(&stdout).unwrap();
subtitles.into_iter().map(|subtitle| (subtitle.meeting_id, subtitle.note_id)).collect()
}
fn main() {
let etherpad_api_key =
std::str::from_utf8(&std::fs::read("/usr/share/etherpad-lite/APIKEY.txt").unwrap())
.unwrap()
.trim()
.to_owned();
let etherpad_api_version = reqwest::blocking::get("http://localhost:9001/api/")
.unwrap()
.json::<EtherpadApiVersion>()
.unwrap()
.current_version;
let meetings_on_disk: HashSet<String> = get_meetings_on_disk();
println!("found {} meetings on disk.", meetings_on_disk.len());
let get_meetings_url = build_api_url("getMeetings", &[]);
let document = reqwest::blocking::get(&get_meetings_url).unwrap().text().unwrap();
let re = regex::Regex::new(r"<internalMeetingID>([0-9a-f]{40}-\d+)</internalMeetingID>").unwrap();
let mut running_meetings : HashSet<String> = HashSet::new();
for id in re.captures_iter(&document) {
running_meetings.insert(id[1].to_owned());
let stored_notes = get_stored_notes(&etherpad_api_version, &etherpad_api_key);
let all_note_pads_by_meeting_id = get_mongo_notes_by_meeting_id();
let all_subtitle_pads_by_meeting_id = get_mongo_subtitles_by_meeting_id();
let all_pads_by_meeting_id = all_note_pads_by_meeting_id.union(&all_subtitle_pads_by_meeting_id);
let running_meetings = get_mongo_running_meetings();
let active_pads : HashSet::<String> =
all_pads_by_meeting_id.filter_map(|(meeting_id, note_id)| {
if running_meetings.contains(meeting_id) {
println!(
"pad {} belongs to running meeting {}",
&note_id, &meeting_id
);
Some(note_id.clone())
} else {
println!("stale pad {}", &note_id);
None
}
}).collect();
let notes_to_remove: HashSet<&str> = stored_notes
.iter()
.map(String::as_str)
.filter_map(|note_id| {
if !active_pads.contains(note_id) {
Some(note_id)
} else {
None
}
})
.collect();
dbg!(notes_to_remove.len());
let kept_notes: Vec<&str> = stored_notes
.iter()
.map(String::as_str)
.filter(|note_id| !notes_to_remove.contains(*note_id))
.map(|note_id| note_id.clone())
.collect();
for note in notes_to_remove {
let res = reqwest::blocking::get(&format!(
"http://localhost:9001/api/{}/deletePad?padID={}&apikey={}",
&etherpad_api_version, &note, &etherpad_api_key
))
.unwrap();
if !res.text().unwrap().contains(r#""message":"ok""#) {
eprintln!("error deleting note {}", &note);
}
}
dbg!(&running_meetings);
println!("found {} running meetings.", running_meetings.len());
let stale_meetings : Vec<String> = meetings_on_disk.difference(&running_meetings).map(|x|{format!("/var/bigbluebutton/{}", x)}).collect();
let stale_meetings: Vec<String> = meetings_on_disk
.difference(&running_meetings)
.map(|x| format!("/var/bigbluebutton/{}", x))
.collect();
println!("found {} stale meetings.", stale_meetings.len());
for dirname in stale_meetings {
print!("deleting {}...", dirname);
std::fs::remove_dir_all(dirname).unwrap();
println!(" removed!");
print!("deleting {}...", dirname);
//std::fs::remove_dir_all(dirname).unwrap();
println!(" removed!");
}
dbg!(&kept_notes);
println!("done cleaning up.");
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment