evobench_tools/run/
polling_pool.rs

1//! Handle polling the upstream project repository for changes, and
2//! also check commit ids for insertions for validity
3
4use std::{collections::BTreeMap, num::NonZero, path::PathBuf, str::FromStr, sync::Arc};
5
6use anyhow::Result;
7use itertools::Itertools;
8use run_git::git::GitWorkingDir;
9
10use crate::{
11    git::GitHash,
12    run::{
13        working_directory::{
14            REMOTE_NAME, WorkingDirectoryAutoCleanOpts, WorkingDirectoryWithPoolMut,
15        },
16        working_directory_pool::WorkingDirectoryPoolContext,
17    },
18    serde_types::{
19        date_and_time::DateTimeWithOffset, git_branch_name::GitBranchName,
20        git_reference::GitReference, git_url::GitUrl,
21    },
22    utillib::arc::CloneArc,
23};
24
25use super::{
26    config::JobTemplate,
27    working_directory_pool::{
28        WorkingDirectoryId, WorkingDirectoryPool, WorkingDirectoryPoolBaseDir,
29    },
30};
31
32fn check_exists(git_working_dir: &GitWorkingDir, commit: &GitHash) -> Result<bool> {
33    let commit_str = commit.to_string();
34    git_working_dir.contains_reference(&commit_str)
35}
36
37pub struct PollingPool {
38    pool: WorkingDirectoryPool,
39}
40
41impl PollingPool {
42    pub fn open(remote_repository_url: GitUrl, polling_pool_base: PathBuf) -> Result<Self> {
43        let base_dir = Arc::new(WorkingDirectoryPoolBaseDir::new(
44            Some(polling_pool_base),
45            &|| unreachable!("no fallback needed as path is always given"),
46        )?);
47        let pool = WorkingDirectoryPool::open(
48            WorkingDirectoryPoolContext {
49                capacity: NonZero::try_from(1).unwrap(),
50                // Provide auto_clean really just to silence the info
51                // message--basically infinite, since we're just using Git
52                // operations, and those should never leark (OK, assuming
53                // git gc runs.)
54                auto_clean: {
55                    let min_age_days = 60; // short enough so that I learn about issues
56                    Some(WorkingDirectoryAutoCleanOpts {
57                        min_age_days,
58                        min_num_runs: 3 * 60 * 24 * usize::from(min_age_days),
59                        // Uh, we're not using jobs here anyway.
60                        wait_until_commit_done: false,
61                    })
62                },
63                remote_repository_url,
64                base_dir,
65                signal_change: None,
66            },
67            true,
68            false,
69        )?
70        .into_inner();
71        Ok(Self { pool })
72    }
73
74    /// Updates the remotes, but only if the commit isn't already in
75    /// the local clone.
76    pub fn commit_is_valid(&mut self, commit: &GitHash) -> Result<bool> {
77        let working_directory_id = {
78            let mut pool = self.pool.lock_mut("PollingPool.commit_is_valid")?;
79            pool.clear_current_working_directory()?;
80            pool.get_first()?
81        };
82        let (res, cleanup) = self.pool.process_in_working_directory(
83            working_directory_id,
84            &DateTimeWithOffset::now(None),
85            |mut working_directory| {
86                let working_directory = working_directory.get().expect("still there");
87                // Check for the commit first, then if it fails, try
88                // to update; both for performance, but also to
89                // minimize contact with issues with remote server.
90                let git_working_dir = &working_directory.git_working_dir;
91                Ok(check_exists(git_working_dir, commit)? || {
92                    let _ = working_directory.fetch(Some(commit))?;
93                    check_exists(git_working_dir, commit)?
94                })
95            },
96            None,
97            &format!("verifying commit {commit}"),
98            None,
99        )?;
100        self.pool.working_directory_cleanup(cleanup)?;
101        Ok(res)
102    }
103
104    /// Get working dir, run git fetch, and return its id for
105    /// subsequent work on it
106    pub fn updated_working_dir(&mut self) -> Result<WorkingDirectoryId> {
107        let working_directory_id = {
108            let mut pool = self.pool.lock_mut("PollingPool.updated_working_dir")?;
109            pool.clear_current_working_directory()?;
110            pool.get_first()?
111        };
112        let (res, cleanup) = self.pool.process_in_working_directory(
113            working_directory_id,
114            &DateTimeWithOffset::now(None),
115            |mut working_directory| {
116                let working_directory = working_directory.get().expect("still there");
117                _ = working_directory.fetch(None)?;
118                Ok(working_directory_id)
119            },
120            None,
121            "updated_working_dir()",
122            None,
123        )?;
124        self.pool.working_directory_cleanup(cleanup)?;
125        Ok(res)
126    }
127
128    /// Currently `working_directory_id` (get it from
129    /// `updated_working_dir`) always represents the same single
130    /// working directory, but maybe that will change?
131    pub fn process_in_working_directory<R>(
132        &mut self,
133        working_directory_id: WorkingDirectoryId,
134        timestamp: &DateTimeWithOffset,
135        action: impl FnOnce(WorkingDirectoryWithPoolMut) -> Result<R>,
136        context: &str,
137    ) -> Result<R> {
138        {
139            let pool = self
140                .pool
141                .lock_mut("PollingPool.process_in_working_directory")?;
142            // XX why again can and do we just clear it everywhere? What
143            // was the status implication in the cleared or not
144            // clearedness?
145            pool.clear_current_working_directory()?;
146        }
147        let (r, token) = self.pool.process_in_working_directory(
148            working_directory_id,
149            timestamp,
150            action,
151            None,
152            context,
153            None,
154        )?;
155        self.pool.working_directory_cleanup(token)?;
156        Ok(r)
157    }
158
159    /// Returns the resolved commit ids for the requested names, and
160    /// any names that failed to resolve.
161    pub fn resolve_branch_names<'b>(
162        &mut self,
163        working_directory_id: WorkingDirectoryId,
164        branch_names: &'b BTreeMap<GitBranchName, Arc<[JobTemplate]>>,
165    ) -> Result<(
166        Vec<(&'b GitBranchName, GitHash, Arc<[JobTemplate]>)>,
167        Vec<String>,
168    )> {
169        self.process_in_working_directory(
170            working_directory_id,
171            &DateTimeWithOffset::now(None),
172            |mut working_directory| {
173                let working_directory = working_directory.get().expect("still there");
174                let git_working_dir = &working_directory.git_working_dir;
175                let mut non_resolving = Vec::new();
176                let mut ids = Vec::new();
177                for (name, job_templates) in branch_names {
178                    let ref_string = name.to_ref_string_in_remote(REMOTE_NAME);
179                    if let Some(id) = git_working_dir.git_rev_parse(&ref_string, true)? {
180                        ids.push((name, GitHash::from_str(&id)?, job_templates.clone_arc()))
181                    } else {
182                        non_resolving.push(ref_string);
183                    }
184                }
185                Ok((ids, non_resolving))
186            },
187            &format!("resolving branch names {}", branch_names.keys().join(", ")),
188        )
189    }
190
191    /// Runs git rev-parse on all references, returning None for those
192    /// that do not resolve. `remote_name`, if given (e.g. "origin"),
193    /// is prefixed to the references with a slash.
194    pub fn resolve_references<R: AsRef<GitReference>>(
195        &mut self,
196        working_directory_id: WorkingDirectoryId,
197        remote_name: Option<&str>,
198        references: impl IntoIterator<Item = R>,
199    ) -> Result<Vec<Option<GitHash>>> {
200        self.process_in_working_directory(
201            working_directory_id,
202            &DateTimeWithOffset::now(None),
203            |mut working_directory| {
204                let working_directory = working_directory.get().expect("still there");
205                let git_working_dir = &working_directory.git_working_dir;
206
207                references
208                    .into_iter()
209                    .map(|reference| -> Result<Option<GitHash>> {
210                        let reference = reference.as_ref();
211                        let reference = reference.as_str();
212                        let tmp;
213                        let full_reference = if let Some(remote_name) = remote_name {
214                            tmp = format!("{remote_name}/{reference}");
215                            &*tmp
216                        } else {
217                            reference
218                        };
219                        Ok(git_working_dir
220                            .git_rev_parse(full_reference, true)?
221                            .map(|s| GitHash::from_str(&s).expect("git always returns git hashes")))
222                    })
223                    .try_collect()
224            },
225            "resolving references",
226        )
227    }
228}
evobench_tools/run/polling_pool.rs

evobench_tools/run/
polling_pool.rs
