libzypp  17.36.7
repodownloaderwf.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 #include "repodownloaderwf.h"
12 
13 #include <utility>
14 #include <zypp-media/ng/Provide>
15 #include <zypp-media/ng/ProvideSpec>
16 #include <zypp/ng/Context>
17 #include <zypp/ng/repo/Downloader>
18 #include <zypp-common/PublicKey.h>
19 #include <zypp/KeyRing.h>
20 
25 
26 // sync workflow helpers
29 
30 #undef ZYPP_BASE_LOGGER_LOGGROUP
31 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::repomanager"
32 
33 
34 namespace zyppng {
35  namespace {
36 
37  using namespace zyppng::operators;
38 
39  template < class Executor, class OpType >
40  struct DownloadMasterIndexLogic : public LogicBase<Executor, OpType>
41  {
42  public:
43  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
44 
45  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
46  using ZyppContextType = typename remove_smart_ptr_t<DlContextRefType>::ContextType;
47  using ProvideType = typename ZyppContextType::ProvideType;
48  using MediaHandle = typename ProvideType::MediaHandle;
49  using ProvideRes = typename ProvideType::Res;
50 
51  DownloadMasterIndexLogic( DlContextRefType &&ctxRef, MediaHandle &&mediaHandle, zypp::filesystem::Pathname &&masterIndex_r )
52  : _dlContext( std::move(ctxRef) )
53  , _media(std::move( mediaHandle ))
54  , _masterIndex(std::move( masterIndex_r ))
55  { }
56 
57  public:
58  MaybeAsyncRef<expected<DlContextRefType>> execute( ) {
59  // always download them, even if repoGpgCheck is disabled
60  _sigpath = _masterIndex.extend( ".asc" );
61  _keypath = _masterIndex.extend( ".key" );
62  _destdir = _dlContext->destDir();
63 
64  auto providerRef = _dlContext->zyppContext()->provider();
65  return provider()->provide( _media, _masterIndex, ProvideFileSpec().setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) )
66  | and_then( [this]( ProvideRes && masterres ) {
67 
68  return std::vector {
69  // fetch signature and keys
70  provider()->provide( _media, _sigpath, ProvideFileSpec().setOptional( true ).setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) )
71  | and_then( ProvideType::copyResultToDest ( provider(), _destdir / _sigpath ) ),
72  provider()->provide( _media, _keypath, ProvideFileSpec().setOptional( true ).setDownloadSize( zypp::ByteCount( 20, zypp::ByteCount::MB ) ) )
73  | and_then( ProvideType::copyResultToDest ( provider(), _destdir / _keypath ) ),
74  }
75  | join()
76  | [this,masterres=std::move(masterres)]( std::vector<expected<zypp::ManagedFile>> &&res ) {
77  // remember downloaded files
78  std::for_each( res.begin (), res.end(),
79  [this]( expected<zypp::ManagedFile> &f){
80  if (f.is_valid () ) {
81  _dlContext->files().push_back( std::move(f.get()));
82  }
83  });
84  return make_expected_success( std::move(masterres) );
85  };
86 
87  } )
88  // execute plugin verification if there is one
89  | and_then( std::bind( &DownloadMasterIndexLogic::pluginVerification, this, std::placeholders::_1 ) )
90 
91  // signature checking
92  | and_then( std::bind( &DownloadMasterIndexLogic::signatureCheck, this, std::placeholders::_1 ) )
93 
94  // copy everything into a directory
95  | and_then( ProvideType::copyResultToDest ( providerRef, _destdir / _masterIndex ) )
96 
97  // final tasks
98  | and_then([this]( zypp::ManagedFile &&masterIndex ) {
99  // Accepted!
100  _dlContext->repoInfo().setMetadataPath( _destdir );
101  _dlContext->repoInfo().setValidRepoSignature( _repoSigValidated );
102 
103  // release the media handle
104  _media = MediaHandle();
105  auto &allFiles = _dlContext->files();
106 
107  // make sure the masterIndex is in front
108  allFiles.insert( allFiles.begin (), std::move(masterIndex) );
109  return make_expected_success( std::move(_dlContext) );
110  });
111  }
112 
113 
114  private:
115  auto provider () {
116  return _dlContext->zyppContext()->provider();
117  }
118 
119  MaybeAsyncRef<expected<ProvideRes>> signatureCheck ( ProvideRes &&res ) {
120 
121  if ( _dlContext->repoInfo().repoGpgCheck() ) {
122 
123  // The local files are in destdir_r, if they were present on the server
124  zypp::Pathname sigpathLocal { _destdir/_sigpath };
125  zypp::Pathname keypathLocal { _destdir/_keypath };
126  bool isSigned = zypp::PathInfo(sigpathLocal).isExist();
127 
128  if ( isSigned || _dlContext->repoInfo().repoGpgCheckIsMandatory() ) {
129 
130  auto verifyCtx = zypp::keyring::VerifyFileContext( res.file() );
131 
132  // only add the signature if it exists
133  if ( isSigned )
134  verifyCtx.signature( sigpathLocal );
135 
136  // only add the key if it exists
137  if ( zypp::PathInfo(keypathLocal).isExist() ) {
138  try {
139  _dlContext->zyppContext()->keyRing()->importKey( zypp::PublicKey(keypathLocal), false );
140  } catch (...) {
142  }
143  }
144 
145  // set the checker context even if the key is not known
146  // (unsigned repo, key file missing; bnc #495977)
147  verifyCtx.keyContext( _dlContext->repoInfo() );
148 
149  return getExtraKeysInRepomd( std::move(res ) )
150  | and_then([this, vCtx = std::move(verifyCtx) ]( ProvideRes &&res ) mutable {
151  for ( const auto &keyData : _buddyKeys ) {
152  DBG << "Keyhint remember buddy " << keyData << std::endl;
153  vCtx.addBuddyKey( keyData.id() );
154  }
155 
156  return SignatureFileCheckWorkflow::verifySignature( _dlContext->zyppContext(), std::move(vCtx))
157  | and_then([ this, res = std::move(res) ]( zypp::keyring::VerifyFileContext verRes ){
158  // remember the validation status
159  _repoSigValidated = verRes.fileValidated();
160  return make_expected_success(std::move(res));
161  });
162  });
163 
164  } else {
165  WAR << "Accept unsigned repository because repoGpgCheck is not mandatory for " << _dlContext->repoInfo().alias() << std::endl;
166  }
167  } else {
168  WAR << "Signature checking disabled in config of repository " << _dlContext->repoInfo().alias() << std::endl;
169  }
171  }
172 
173  // execute the repo verification if there is one
174  expected<ProvideRes> pluginVerification ( ProvideRes &&prevRes ) {
175  // The local files are in destdir_r, if they were present on the server
176  zypp::Pathname sigpathLocal { _destdir/_sigpath };
177  zypp::Pathname keypathLocal { _destdir/_keypath };
178  if ( _dlContext->pluginRepoverification() && _dlContext->pluginRepoverification()->isNeeded() ) {
179  try {
180  _dlContext->pluginRepoverification()->getChecker( sigpathLocal, keypathLocal, _dlContext->repoInfo() )( prevRes.file() );
181  } catch ( ... ) {
182  return expected<ProvideRes>::error( std::current_exception () );
183  }
184  }
185  return make_expected_success(std::move(prevRes));
186  }
187 
192  MaybeAsyncRef<expected<ProvideRes>> getExtraKeysInRepomd ( ProvideRes &&res ) {
193 
194  if ( _masterIndex.basename() != "repomd.xml" ) {
195  return makeReadyResult( expected<ProvideRes>::success( std::move(res) ) );
196  }
197 
198  std::vector<std::pair<std::string,std::string>> keyhints { zypp::parser::yum::RepomdFileReader(res.file()).keyhints() };
199  if ( keyhints.empty() )
200  return makeReadyResult( expected<ProvideRes>::success( std::move(res) ) );
201  DBG << "Check keyhints: " << keyhints.size() << std::endl;
202 
203  auto keyRing { _dlContext->zyppContext()->keyRing() };
204  return zypp::parser::yum::RepomdFileReader(res.file()).keyhints()
205  | transform([this, keyRing]( std::pair<std::string, std::string> val ) {
206 
207  const auto& [ file, keyid ] = val;
208  auto keyData = keyRing->trustedPublicKeyData( keyid );
209  if ( keyData ) {
210  DBG << "Keyhint is already trusted: " << keyid << " (" << file << ")" << std::endl;
211  return makeReadyResult ( expected<zypp::PublicKeyData>::success(keyData) ); // already a trusted key
212  }
213 
214  DBG << "Keyhint search key " << keyid << " (" << file << ")" << std::endl;
215 
216  keyData = keyRing->publicKeyData( keyid );
217  if ( keyData )
219 
220  // TODO: Enhance the key caching in general...
221  const zypp::ZConfig & conf = _dlContext->zyppContext()->config();
222  zypp::Pathname cacheFile = conf.repoManagerRoot() / conf.pubkeyCachePath() / file;
223 
224  return zypp::PublicKey::noThrow(cacheFile)
225  | [ keyid = keyid ]( auto &&key ){
226  if ( key.fileProvidesKey( keyid ) )
227  return make_expected_success( std::forward<decltype(key)>(key) );
228  else
229  return expected<zypp::PublicKey>::error( std::make_exception_ptr (zypp::Exception("File does not provide key")));
230  }
231  | or_else ([ this, file = file, keyid = keyid, cacheFile ] ( auto ) mutable -> MaybeAsyncRef<expected<zypp::PublicKey>> {
232  auto providerRef = _dlContext->zyppContext()->provider();
233  return providerRef->provide( _media, file, ProvideFileSpec().setOptional(true) )
234  | and_then( ProvideType::copyResultToDest( providerRef, _destdir / file ) )
235  | and_then( [this, providerRef, file, keyid , cacheFile = std::move(cacheFile)]( zypp::ManagedFile &&res ) {
236 
237  // remember we downloaded the file
238  _dlContext->files().push_back ( std::move(res) );
239 
240  auto key = zypp::PublicKey::noThrow( _dlContext->files().back() );
241  if ( not key.fileProvidesKey( keyid ) ) {
242  const std::string str = (zypp::str::Str() << "Keyhint " << file << " does not contain a key with id " << keyid << ". Skipping it.");
243  WAR << str << std::endl;
244  return makeReadyResult(expected<zypp::PublicKey>::error( std::make_exception_ptr( zypp::Exception(str)) ));
245  }
246 
247  // Try to cache it...
248  zypp::filesystem::assert_dir( cacheFile.dirname() );
249  return providerRef->copyFile( key.path(), cacheFile )
250  | [ key ]( expected<zypp::ManagedFile> res ) mutable {
251  if ( res ) {
252  // do not delete from cache
253  res->resetDispose ();
254  }
255  return expected<zypp::PublicKey>::success( std::move(key) );
256  };
257  });
258  })
259  | and_then( [ keyRing, keyid = keyid ]( zypp::PublicKey key ){
260  keyRing->importKey( key, false ); // store in general keyring (not trusted!)
261  return expected<zypp::PublicKeyData>::success(keyRing->publicKeyData( keyid )); // fetch back from keyring in case it was a hidden key
262  });
263  })
264  | [this, res = res] ( std::vector<expected<zypp::PublicKeyData>> &&keyHints ) mutable {
265  std::for_each( keyHints.begin(), keyHints.end(), [this]( expected<zypp::PublicKeyData> &keyData ){
266  if ( keyData && *keyData ) {
267  if ( not zypp::PublicKey::isSafeKeyId( keyData->id() ) ) {
268  WAR << "Keyhint " << keyData->id() << " for " << *keyData << " is not strong enough for auto import. Just caching it." << std::endl;
269  return;
270  }
271  _buddyKeys.push_back ( std::move(keyData.get()) );
272  }
273  });
274 
275  MIL << "Check keyhints done. Buddy keys: " << _buddyKeys.size() << std::endl;
276  return expected<ProvideRes>::success (std::move(res));
277  };
278  }
279 
280  DlContextRefType _dlContext;
281  MediaHandle _media;
283 
287  zypp::TriBool _repoSigValidated = zypp::indeterminate;
288 
289  std::vector<zypp::PublicKeyData> _buddyKeys;
290  };
291 
292  }
293 
295  {
296  return SimpleExecutor<DownloadMasterIndexLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>>>::run( std::move(dl), std::move(mediaHandle), std::move(masterIndex_r) );
297  }
298 
300  {
301  return SimpleExecutor<DownloadMasterIndexLogic, SyncOp<expected<repo::SyncDownloadContextRef>>>::run( std::move(dl), std::move(mediaHandle), std::move(masterIndex_r) );
302  }
303 
305  {
306  using namespace zyppng::operators;
307  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
308  | and_then([ dl, mi = std::move(masterIndex_r) ]( ProvideMediaHandle handle ) mutable {
309  return downloadMasterIndex( std::move(dl), std::move(handle), std::move(mi) );
310  });
311  }
312 
314  {
315  using namespace zyppng::operators;
316  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
317  | and_then([ dl, mi = std::move(masterIndex_r) ]( SyncMediaHandle handle ) mutable {
318  return downloadMasterIndex( std::move(dl), std::move(handle), std::move(mi) );
319  });
320  }
321 
322 
323  namespace {
324  template <class DlContextRefType, class MediaHandleType>
325  auto statusImpl ( DlContextRefType dlCtx, MediaHandleType &&mediaHandle ) {
326 
327  constexpr bool isAsync = std::is_same_v<DlContextRefType,repo::AsyncDownloadContextRef>;
328 
329  const auto finalizeStatus = [ dlCtx ]( zypp::RepoStatus status ){
330  return expected<zypp::RepoStatus>::success( zypp::RepoStatus( dlCtx->repoInfo()) && status );
331  };
332 
333  switch( dlCtx->repoInfo().type().toEnum()) {
335  return RpmmdWorkflows::repoStatus( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
337  return SuseTagsWorkflows::repoStatus( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
339  return PlaindirWorkflows::repoStatus ( dlCtx, std::forward<MediaHandleType>(mediaHandle) ) | and_then( std::move(finalizeStatus) );
341  break;
342  }
343 
344  return makeReadyResult<expected<zypp::RepoStatus>, isAsync >( expected<zypp::RepoStatus>::error( ZYPP_EXCPT_PTR (zypp::repo::RepoUnknownTypeException(dlCtx->repoInfo()))) );
345  }
346  }
347 
349  return statusImpl( dl, std::move(mediaHandle) );
350  }
351 
352  expected<zypp::RepoStatus> RepoDownloaderWorkflow::repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle) {
353  return statusImpl( dl, std::move(mediaHandle) );
354  }
355 
357  using namespace zyppng::operators;
358  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
359  | and_then([ dl ]( ProvideMediaHandle handle ) {
360  return repoStatus( dl, std::move(handle) );
361  });
362  }
363 
365  using namespace zyppng::operators;
366  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
367  | and_then([ dl ]( SyncMediaHandle handle ) {
368  return repoStatus( dl, std::move(handle) );
369  });
370  }
371 
372 
373  namespace {
374  template <class DlContextRefType, class MediaHandleType>
375  auto downloadImpl ( DlContextRefType dlCtx, MediaHandleType &&mediaHandle, ProgressObserverRef &&progressObserver ) {
376 
377  constexpr bool isAsync = std::is_same_v<DlContextRefType,repo::AsyncDownloadContextRef>;
378 
379  switch( dlCtx->repoInfo().type().toEnum()) {
381  return RpmmdWorkflows::download( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle), std::move(progressObserver) );
383  return SuseTagsWorkflows::download( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle), std::move(progressObserver) );
385  return PlaindirWorkflows::download ( std::move(dlCtx), std::forward<MediaHandleType>(mediaHandle) );
387  break;
388  }
389 
390  return makeReadyResult<expected<DlContextRefType>, isAsync >( expected<DlContextRefType>::error( ZYPP_EXCPT_PTR (zypp::repo::RepoUnknownTypeException(dlCtx->repoInfo()))) );
391  }
392  }
393 
394  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > RepoDownloaderWorkflow::download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
395  {
396  return downloadImpl( dl, std::move(mediaHandle), std::move(progressObserver) );
397  }
398 
399  expected<repo::SyncDownloadContextRef> RepoDownloaderWorkflow::download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
400  {
401  return downloadImpl( dl, std::move(mediaHandle), std::move(progressObserver) );
402  }
403 
404  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > RepoDownloaderWorkflow::download(repo::AsyncDownloadContextRef dl, AsyncLazyMediaHandle mediaHandle, ProgressObserverRef progressObserver)
405  {
406  using namespace zyppng::operators;
407  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
408  | and_then([ dl, po = std::move(progressObserver) ]( ProvideMediaHandle handle ) mutable {
409  return downloadImpl( dl, std::move(handle), std::move(po) );
410  });
411  }
412 
413  expected<repo::SyncDownloadContextRef> RepoDownloaderWorkflow::download(repo::SyncDownloadContextRef dl, SyncLazyMediaHandle mediaHandle, ProgressObserverRef progressObserver)
414  {
415  using namespace zyppng::operators;
416  return dl->zyppContext()->provider()->attachMediaIfNeeded( mediaHandle )
417  | and_then([ dl, po = std::move(progressObserver) ]( SyncMediaHandle handle ) mutable {
418  return downloadImpl( dl, std::move(handle), std::move(po) );
419  });
420  }
421 
422 }
#define MIL
Definition: Logger.h:100
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver=nullptr)
boost::logic::tribool TriBool
3-state boolean logic (true, false and indeterminate).
Definition: String.h:30
thrown when it was impossible to determine this repo type.
auto transform(Transformation &&transformation)
Definition: transform.h:70
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:324
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: susetags.cc:330
Store and operate with byte count.
Definition: ByteCount.h:31
Pathname pubkeyCachePath() const
Path where the pubkey caches.
Definition: ZConfig.cc:1075
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:175
zypp::Pathname _masterIndex
zypp::Pathname _keypath
String related utilities and Regular expression matching.
zypp::TriBool _repoSigValidated
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition: expected.h:397
I/O context for KeyRing::verifyFileSignatureWorkflow.
static const Unit MB
1000^2 Byte
Definition: ByteCount.h:61
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
zypp::Pathname _sigpath
#define ZYPP_EXCPT_PTR(EXCPT)
Drops a logline and returns Exception as a std::exception_ptr.
Definition: Exception.h:428
Pathname repoManagerRoot() const
The RepoManager root directory.
Definition: ZConfig.cc:980
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: plaindir.cc:88
MediaHandle _media
auto or_else(Fun &&function)
Definition: expected.h:630
const Pathname & signature() const
Detached signature or empty.
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:286
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:126
Interim helper class to collect global options and settings.
Definition: ZConfig.h:68
#define WAR
Definition: Logger.h:101
auto join()
Definition: wait.h:133
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition: logichelpers.h:223
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: plaindir.cc:42
DlContextRefType _dlContext
typename conditional< B, T, F >::type conditional_t
Definition: TypeTraits.h:39
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition: asyncop.h:297
static expected success(ConsParams &&...params)
Definition: expected.h:115
std::shared_ptr< AsyncOp< T > > AsyncOpRef
Definition: asyncop.h:255
Reads through a repomd.xml file and collects type, location, checksum and other data about metadata f...
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Base class for Exception.
Definition: Exception.h:146
expected< zypp::keyring::VerifyFileContext > verifySignature(SyncContextRef ctx, zypp::keyring::VerifyFileContext context)
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: rpmmd.cc:74
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: susetags.cc:84
auto and_then(Fun &&function)
Definition: expected.h:623
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
Interface of repomd.xml file reader.
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: rpmmd.cc:171
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition: expected.h:423
Track changing files or directories.
Definition: RepoStatus.h:40
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition: Exception.h:436
#define DBG
Definition: Logger.h:99
zypp::Pathname _destdir
std::vector< zypp::PublicKeyData > _buddyKeys