23 inline bool preloadEnabled()
25 const char *val = ::getenv(
"ZYPP_PCK_PRELOAD");
31 return ( std::string_view( val ) ==
"1" );
35 if ( pck.isKind<Package>() ) {
37 }
else if ( pck.isKind<SrcPackage>() ) {
84 MIL <<
"No more jobs pending, exiting worker" << std::endl;
100 userData.
set(
"description",
_(
"No mirror found") );
106 ERR <<
"Failed to create target dir for file: " <<
_targetPath << std::endl;
108 userData.
set(
"description",
_(
"Could not create target File") );
123 if ( !pathInfo.
isFile() ) {
124 if ( pathInfo.
isDir () )
133 userData.
set(
"Url", url );
134 userData.
set(
"description",
_(
"Already in Cache") );
151 _req = std::make_shared<zyppng::NetworkRequest>( url,
_tmpFile );
156 _req->resetRequestRanges();
164 _req->transferSettings() = settings;
177 const auto &pi =
_job;
226 std::vector<RepoUrl>::iterator curr = repoDlInfo._baseUrls.end();
227 int currentSmallestRef = INT_MAX;
229 for (
auto i = repoDlInfo._baseUrls.begin(); i != repoDlInfo._baseUrls.end(); i++ ) {
230 auto mirrorPtr = &(*i);
232 if ( skip == mirrorPtr )
240 if ( ( i->refs + i->miss ) < currentSmallestRef ) {
241 currentSmallestRef = ( i->refs + i->miss );
246 if ( curr == repoDlInfo._baseUrls.end() )
252 MIL <<
"Request for " << req.
url() <<
" started" << std::endl;
260 userData.
set(
"Url",
_req->url() );
275 MIL <<
"Request for " << req.
url() <<
" finished. (" << err.
toString() <<
")" << std::endl;
282 userData.
set(
"Url", req.
url() );
283 userData.
set(
"description",
_(
"Finished") );
288 const auto &error = req.
error();
289 switch ( error.type() ) {
313 std::string authHint = error.extraInfoValue(
"authHint", std::string());
328 MIL <<
"Download from mirror failed for file " << req.
url () <<
" trying to taint mirror and move on" << std::endl;
337 MIL <<
"Found new mirror: " << url <<
" recovering, retry count: " <<
_notFoundRetry << std::endl;
340 _req->transferSettings () = settings;
351 DBG <<
"BUG: Download error flag is set , but Error code is NoError" << std::endl;
362 userData.
set(
"Url", url );
364 userData.
set(
"description",
_(
"Already in Cache") );
380 if ( settings.
proxy().empty() )
396 std::string header { el.first };
399 MIL <<
"Added custom header -> " << header << std::endl;
404 resultSet = settings;
410 zyppng::NetworkRequestRef
_req;
434 if ( !preloadEnabled() ) {
435 MIL <<
"CommitPackagePreloader disabled" << std::endl;
441 MIL <<
"No receiver for the CommitPreloadReport, skipping preload phase" << std::endl;
446 _dispatcher = std::make_shared<zyppng::NetworkRequestDispatcher>();
453 _pTracker = std::make_shared<internal::ProgressTracker>();
463 for (
const auto &step : steps ) {
464 switch ( step.stepType() )
483 if ( pi->lookupLocation().checksum().empty() )
487 if( !pckCachedLocation(pi).empty() )
490 auto repoDlsIter =
_dlRepoInfo.find( pi.repository().id() );
495 ERR <<
"Failed to create predownload cache for repo " << pi.repoInfo().alias() << std::endl;
500 std::vector<RepoUrl> repoUrls;
501 const auto bu = pi.repoInfo().baseUrls();
502 std::for_each( bu.begin(), bu.end(), [&](
const zypp::Url &u ) {
505 MIL <<
"Trying scheme '" << url.
getScheme() <<
"'" << std::endl;
512 .headers = std::move(custom_headers)
517 if( repoUrls.empty() ) {
518 MIL <<
"Skipping predownload for " << step.satSolvable() <<
" no downloading URL" << std::endl;
524 if ( repoUrls.begin()->baseUrl.getHost() ==
"download.opensuse.org" ){
529 pi.repository().id(),
547 const auto &workerDone = [&,
this](){
548 if ( std::all_of(
_workers.begin(),
_workers.end(), [](
const auto &w ) {
return w->finished();} ) )
564 auto worker = std::make_shared<PreloadWorker>(*this);
565 worker->sigWorkerFinished().connect(workerDone);
567 _workers.push_back( std::move(worker) );
571 MIL <<
"Running preload event loop!" << std::endl;
575 MIL <<
"Preloading done, returning" << std::endl;
580 if ( !preloadEnabled() ) {
581 MIL <<
"CommitPackagePreloader disabled" << std::endl;
600 userData.
set(
"dbps_avg" , static_cast<double>(
_pTracker->_drateTotal ) );
601 userData.
set(
"dbps_current", static_cast<double>(
_pTracker->_drateLast ) );
602 userData.
set(
"bytesReceived", static_cast<double>(
_pTracker->_dnlNow ) );
603 userData.
set(
"bytesRequired", static_cast<double>(
_pTracker->_dnlTotal ) );
std::string getScheme() const
Returns the scheme name of the URL.
RepoUrl * findUsableMirror(RepoUrl *skip=nullptr, bool allowTainted=true)
Tries to find a usable mirror
Repository::IdType _currentRepoId
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
Pathname cachedLocation(const OnMediaLocation &loc_r, const RepoInfo &repo_r)
void onRequestStarted(zyppng::NetworkRequest &req)
[M] Install(multiversion) item (
static ZConfig & instance()
Singleton ctor.
int clean_dir(const Pathname &path)
Like 'rm -r DIR/ *'.
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
zyppng::NetworkRequestRef _req
void appendPathName(const Pathname &path_r, EEncoding eflag_r=zypp::url::E_DECODED)
Extend the path name.
void reset()
Reset to default Ctor values.
Store and operate with byte count.
Url clearQueryString(const Url &url)
static const RepoIdType noRepoId(0)
Id to denote Repo::noRepository.
CommitPackagePreloader & _parent
zyppng::Signal< void()> _sigFinished
std::set< RepoUrl * > _taintedMirrors
std::string toString() const
toString Returns a string representation of the error
media::UrlResolverPlugin::HeaderList headers
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
media::UrlResolverPlugin::HeaderList headers
zyppng::NetworkRequestDispatcherRef _dispatcher
std::string basename() const
Return the last component of this path.
TransferSettings & transferSettings()
callback::SendReport< media::CommitPreloadReport > _report
void preloadTransaction(const std::vector< sat::Transaction::Step > &steps)
OnMediaLocation lookupLocation() const
bool hasError() const
Checks if there was a error with the request.
void failCurrentJob(const zypp::Pathname &localPath, const std::optional< zypp::Url > &url, media::CommitPreloadReport::Error e, const std::optional< std::string > &errorMessage)
bool isKind(const Resolvable::constPtr &p)
Test whether a Resolvable::Ptr is of a certain Kind.
void onRequestFinished(zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)
bool taintCurrentMirror()
Taints the current mirror, returns true if a alternative was found
void makeJobUrl(zypp::Url &resultUrl, media::TransferSettings &resultSet)
zypp::Pathname _targetPath
ByteCount _downloadedBytes
bool set(const std::string &key_r, AnyType val_r)
Set the value for key (nonconst version always returns true).
zyppng::Ref< internal::ProgressTracker > _pTracker
bool isExist() const
Return whether valid stat info exists.
std::vector< zyppng::Ref< PreloadWorker > > _workers
RepoInfo repoInfo() const
Pathname dirname() const
Return all but the last component od this path.
void onRequestProgress(zyppng::NetworkRequest &req, zypp::ByteCount count)
RepoInfo info() const
Return any associated RepoInfo.
The NetworkRequestError class Represents a error that occured in.
NetworkRequestError error() const
Returns the last set Error.
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
std::deque< PoolItem > _requiredDls
sat::detail::RepoIdType IdType
std::map< Repository::IdType, RepoDownloadData > _dlRepoInfo
std::vector< RepoUrl > _baseUrls
int unlink(const Pathname &path)
Like 'unlink'.
Pathname predownloadPath() const
Path where this repo packages are predownloaded.
zyppng::SignalProxy< void()> sigWorkerFinished()
bool any_of(const Container &c, Fnc &&cb)
Typesafe passing of user data via callbacks.
Wrapper class for ::stat/::lstat.
void reportBytesDownloaded(ByteCount newBytes)
Combining sat::Solvable and ResStatus.
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
int rename(const Pathname &oldpath, const Pathname &newpath)
Like 'rename'.
IdType id() const
Expert backdoor.
Easy-to use interface to the ZYPP dependency resolver.
int rmdir(const Pathname &path)
Like 'rmdir'.
Repository repository() const
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
PreloadWorker(CommitPackagePreloader &parent)
static ManagedFile asManagedFile()
Create a temporary file and convert it to a automatically cleaned up ManagedFile. ...