00001
00002
00003
00004
00005
00006
00007
00008 #include "unbunAndRegPhyBunfile.h"
00009 #include "apiHeaderAll.h"
00010 #include "miscServerFunct.h"
00011 #include "objMetaOpr.h"
00012 #include "resource.h"
00013 #include "dataObjOpr.h"
00014 #include "physPath.h"
00015 #include "rcGlobalExtern.h"
00016 #include "reGlobalsExtern.h"
00017
00018 #include "eirods_stacktrace.h"
00019
00020
00021
00022 #include "eirods_resource_backport.h"
00023
00024 int
00025 rsUnbunAndRegPhyBunfile (rsComm_t *rsComm, dataObjInp_t *dataObjInp)
00026 {
00027 int status;
00028 char *rescName;
00029
00030 if ((rescName = getValByKey (&dataObjInp->condInput, DEST_RESC_NAME_KW))
00031 == NULL) {
00032 return USER_NO_RESC_INPUT_ERR;
00033 }
00034 #if 0 // JMC - legacy resource
00035 rescGrpInfo_t *rescGrpInfo = NULL;
00036 status = resolveAndQueResc (rescName, NULL, &rescGrpInfo);
00037 if (status < 0|| NULL == rescGrpInfo) {
00038 rodsLog (LOG_NOTICE,
00039 "rsUnbunAndRegPhyBunfile: resolveAndQueRescerror for %s, status = %d",
00040 rescName, status);
00041 return (status);
00042 }
00043 #endif // JMC - legacy resource
00044
00045 rescInfo_t* rescInfo = new rescInfo_t;
00046 eirods::error err = eirods::get_resc_info( rescName, *rescInfo );
00047 if( !err.ok() ) {
00048 delete rescInfo;
00049 std::stringstream msg;
00050 msg << "rsUnbunAndRegPhyBunfile - failed for [";
00051 msg << rescName;
00052 msg << "]";
00053 eirods::log( PASS( false, -1, msg.str(), err ) );
00054 return -1;
00055 }
00056 status = _rsUnbunAndRegPhyBunfile( rsComm, dataObjInp, rescInfo );
00057
00058
00059 return (status);
00060 }
00061
00062 int
00063 _rsUnbunAndRegPhyBunfile (rsComm_t *rsComm, dataObjInp_t *dataObjInp,
00064 rescInfo_t *rescInfo)
00065 {
00066 int status;
00067 int remoteFlag;
00068 rodsServerHost_t *rodsServerHost;
00069 char *bunFilePath;
00070 char phyBunDir[MAX_NAME_LEN];
00071 int rmBunCopyFlag;
00072 char *dataType = NULL;
00073
00074 char* resc_hier = getValByKey( &dataObjInp->condInput, RESC_HIER_STR_KW );
00075 if( !resc_hier ) {
00076 rodsLog( LOG_NOTICE, "_rsUnbunAndRegPhyBunfile - RESC_HIER_STR_KW is NULL" );
00077 return -1;
00078 }
00079
00080
00081
00082 std::string location;
00083 eirods::error ret = eirods::get_loc_for_hier_string( resc_hier, location );
00084 if( !ret.ok() ) {
00085 eirods::log( PASSMSG( "_rsUnbunAndRegPhyBunfile - failed in get_loc_for_hier_String", ret ) );
00086 return -1;
00087 }
00088
00089
00090 rodsHostAddr_t addr;
00091 memset (&addr, 0, sizeof (addr));
00092 rstrcpy( addr.hostAddr, location.c_str(), NAME_LEN );
00093 remoteFlag = resolveHost( &addr, &rodsServerHost );
00094 if (remoteFlag == REMOTE_HOST) {
00095 addKeyVal (&dataObjInp->condInput, DEST_RESC_NAME_KW,
00096 rescInfo->rescName);
00097 status = remoteUnbunAndRegPhyBunfile (rsComm, dataObjInp,
00098 rodsServerHost);
00099 return status;
00100 }
00101
00102 if ((bunFilePath = getValByKey (&dataObjInp->condInput, BUN_FILE_PATH_KW))
00103 == NULL) {
00104 rodsLog (LOG_ERROR,
00105 "_rsUnbunAndRegPhyBunfile: No filePath input for %s",
00106 dataObjInp->objPath);
00107 return (SYS_INVALID_FILE_PATH);
00108 }
00109
00110 createPhyBundleDir (rsComm, bunFilePath, phyBunDir);
00111 dataType = getValByKey (&dataObjInp->condInput, DATA_TYPE_KW);
00112 status = unbunPhyBunFile (rsComm, dataObjInp->objPath, rescInfo, bunFilePath, phyBunDir,
00113 dataType, 0, resc_hier );
00114
00115 if (status < 0) {
00116 rodsLog (LOG_ERROR,
00117 "_rsUnbunAndRegPhyBunfile:unbunPhyBunFile err for %s to dir %s.stat=%d",
00118 bunFilePath, phyBunDir, status);
00119 return status;
00120 }
00121
00122 if (getValByKey (&dataObjInp->condInput, RM_BUN_COPY_KW) == NULL) {
00123 rmBunCopyFlag = 0;
00124 } else {
00125 rmBunCopyFlag = 1;
00126 }
00127
00128 status = regUnbunPhySubfiles (rsComm, rescInfo, phyBunDir, rmBunCopyFlag);
00129
00130 if (status == CAT_NO_ROWS_FOUND) {
00131
00132 status = 0;
00133 } else if (status < 0) {
00134 rodsLog (LOG_ERROR,
00135 "_rsUnbunAndRegPhyBunfile: regUnbunPhySubfiles for dir %s. stat = %d",
00136 phyBunDir, status);
00137 }
00138
00139 return status;
00140 }
00141
00142 int
00143 regUnbunPhySubfiles (rsComm_t *rsComm, rescInfo_t *rescInfo, char *phyBunDir,
00144 int rmBunCopyFlag)
00145 {
00146 #ifndef USE_BOOST_FS
00147 DIR *dirPtr;
00148 struct dirent *myDirent;
00149 struct stat statbuf;
00150 #endif
00151 char subfilePath[MAX_NAME_LEN];
00152 dataObjInp_t dataObjInp;
00153 dataObjInp_t dataObjUnlinkInp;
00154 int status;
00155 int savedStatus = 0;
00156
00157 dataObjInfo_t *dataObjInfoHead = NULL;
00158 dataObjInfo_t *bunDataObjInfo= NULL;
00159 #ifdef USE_BOOST_FS
00160 path srcDirPath (phyBunDir);
00161 if (!exists(srcDirPath) || !is_directory(srcDirPath)) {
00162 #else
00163 dirPtr = opendir (phyBunDir);
00164 if (dirPtr == NULL) {
00165 #endif
00166 rodsLog (LOG_ERROR,
00167 "regUnbunphySubfiles: opendir error for %s, errno = %d",
00168 phyBunDir, errno);
00169 return (UNIX_FILE_OPENDIR_ERR - errno);
00170 }
00171 bzero (&dataObjInp, sizeof (dataObjInp));
00172 if (rmBunCopyFlag > 0) {
00173 bzero (&dataObjUnlinkInp, sizeof (dataObjUnlinkInp));
00174 addKeyVal (&dataObjUnlinkInp.condInput, IRODS_ADMIN_KW, "");
00175 }
00176
00177 #ifdef USE_BOOST_FS
00178 directory_iterator end_itr;
00179 for (directory_iterator itr(srcDirPath); itr != end_itr;++itr) {
00180 path p = itr->path();
00181 snprintf (subfilePath, MAX_NAME_LEN, "%s",
00182 p.c_str ());
00183 #else
00184 while ((myDirent = readdir (dirPtr)) != NULL) {
00185 if (strcmp (myDirent->d_name, ".") == 0 ||
00186 strcmp (myDirent->d_name, "..") == 0) {
00187 continue;
00188 }
00189 snprintf (subfilePath, MAX_NAME_LEN, "%s/%s",
00190 phyBunDir, myDirent->d_name);
00191 #endif
00192
00193 #ifdef USE_BOOST_FS
00194 #if 0
00195 path p (subfilePath);
00196 #endif
00197 if (!exists(p)) {
00198 #else
00199 status = stat (subfilePath, &statbuf);
00200 if (status != 0) {
00201 closedir (dirPtr);
00202 #endif
00203 rodsLog (LOG_ERROR,
00204 "regUnbunphySubfiles: stat error for %s, errno = %d",
00205 subfilePath, errno);
00206 return (UNIX_FILE_STAT_ERR - errno);
00207 }
00208
00209 #ifdef USE_BOOST_FS
00210 if (!is_regular_file(p)) continue;
00211
00212 path childPath = p.filename();
00213
00214 addKeyVal (&dataObjInp.condInput, QUERY_BY_DATA_ID_KW,
00215 (char *) childPath.c_str ());
00216 #else
00217 if ((statbuf.st_mode & S_IFREG) == 0) continue;
00218
00219
00220 addKeyVal (&dataObjInp.condInput, QUERY_BY_DATA_ID_KW,
00221 myDirent->d_name);
00222 #endif
00223 status = getDataObjInfo (rsComm, &dataObjInp, &dataObjInfoHead,
00224 NULL, 1);
00225 if (status < 0) {
00226 rodsLog (LOG_DEBUG,
00227 "regUnbunphySubfiles: getDataObjInfo error for %s, status = %d",
00228 subfilePath, status);
00229
00230 unlink (subfilePath);
00231 continue;
00232 }
00233 requeDataObjInfoByResc (&dataObjInfoHead, BUNDLE_RESC, 1, 1);
00234 bunDataObjInfo = NULL;
00235 if (strcmp (dataObjInfoHead->rescName, BUNDLE_RESC) != 0) {
00236
00237 rodsLog (LOG_DEBUG,
00238 "regUnbunphySubfiles: No copy in BUNDLE_RESC for %s",
00239 dataObjInfoHead->objPath);
00240
00241 unlink (subfilePath);
00242 continue;
00243 } else {
00244 bunDataObjInfo = dataObjInfoHead;
00245 }
00246 requeDataObjInfoByResc (&dataObjInfoHead, rescInfo->rescName, 1, 1);
00247
00248 if (strcmp (dataObjInfoHead->rescName, rescInfo->rescName) != 0) {
00249
00250 status = regPhySubFile (rsComm, subfilePath, bunDataObjInfo,
00251 rescInfo);
00252 unlink (subfilePath);
00253 if (status < 0) {
00254 rodsLog (LOG_DEBUG,
00255 "regUnbunphySubfiles: regPhySubFile err for %s, status = %d",
00256 bunDataObjInfo->objPath, status);
00257 }
00258 } else {
00259
00260 unlink (subfilePath);
00261 }
00262 if (rmBunCopyFlag > 0) {
00263 rstrcpy (dataObjUnlinkInp.objPath, bunDataObjInfo->objPath,
00264 MAX_NAME_LEN);
00265 status = dataObjUnlinkS (rsComm, &dataObjUnlinkInp, bunDataObjInfo);
00266 if (status < 0) {
00267 rodsLog (LOG_ERROR,
00268 "regUnbunphySubfiles: dataObjUnlinkS err for %s, status = %d",
00269 bunDataObjInfo->objPath, status);
00270 savedStatus = status;
00271 }
00272 }
00273 freeAllDataObjInfo (dataObjInfoHead);
00274
00275 }
00276 clearKeyVal (&dataObjInp.condInput);
00277 #ifndef USE_BOOST_FS
00278 closedir (dirPtr);
00279 #endif
00280 if (status >= 0 && savedStatus < 0) {
00281 return savedStatus;
00282 } else {
00283 return status;
00284 }
00285 }
00286
00287 int
00288 regPhySubFile (rsComm_t *rsComm, char *subfilePath,
00289 dataObjInfo_t *bunDataObjInfo, rescInfo_t *rescInfo)
00290 {
00291 dataObjInfo_t stageDataObjInfo;
00292 dataObjInp_t dataObjInp;
00293 #ifndef USE_BOOST_FS
00294 struct stat statbuf;
00295 #endif
00296 int status;
00297 regReplica_t regReplicaInp;
00298
00299 bzero (&dataObjInp, sizeof (dataObjInp));
00300 bzero (&stageDataObjInfo, sizeof (stageDataObjInfo));
00301 rstrcpy (dataObjInp.objPath, bunDataObjInfo->objPath, MAX_NAME_LEN);
00302 rstrcpy (stageDataObjInfo.objPath, bunDataObjInfo->objPath, MAX_NAME_LEN);
00303 rstrcpy (stageDataObjInfo.rescName, rescInfo->rescName, NAME_LEN);
00304 stageDataObjInfo.rescInfo = rescInfo;
00305
00306 status = getFilePathName (rsComm, &stageDataObjInfo, &dataObjInp);
00307 if (status < 0) {
00308 rodsLog (LOG_ERROR,
00309 "regPhySubFile: getFilePathName err for %s. status = %d",
00310 dataObjInp.objPath, status);
00311 return (status);
00312 }
00313
00314 #ifdef USE_BOOST_FS
00315 path p (stageDataObjInfo.filePath);
00316 if (exists (p)) {
00317 #else
00318 status = stat (stageDataObjInfo.filePath, &statbuf);
00319 if (status == 0 || errno != ENOENT) {
00320 #endif
00321 status = resolveDupFilePath (rsComm, &stageDataObjInfo, &dataObjInp);
00322 if (status < 0) {
00323 rodsLog (LOG_ERROR,
00324 "regPhySubFile: resolveDupFilePath err for %s. status = %d",
00325 stageDataObjInfo.filePath, status);
00326 return (status);
00327 }
00328 }
00329
00330 mkDirForFilePath (rsComm, "/", stageDataObjInfo.filePath, getDefDirMode ());
00331
00332 status = link (subfilePath, stageDataObjInfo.filePath);
00333 if (status < 0) {
00334 rodsLog (LOG_ERROR,
00335 "regPhySubFile: link error %s to %s. errno = %d",
00336 subfilePath, stageDataObjInfo.filePath, errno);
00337 return (UNIX_FILE_LINK_ERR - errno);
00338 }
00339
00340 bzero (®ReplicaInp, sizeof (regReplicaInp));
00341 regReplicaInp.srcDataObjInfo = bunDataObjInfo;
00342 regReplicaInp.destDataObjInfo = &stageDataObjInfo;
00343 addKeyVal (®ReplicaInp.condInput, SU_CLIENT_USER_KW, "");
00344 addKeyVal (®ReplicaInp.condInput, IRODS_ADMIN_KW, "");
00345
00346 status = rsRegReplica (rsComm, ®ReplicaInp);
00347
00348 clearKeyVal (®ReplicaInp.condInput);
00349 if (status < 0) {
00350 rodsLog (LOG_ERROR,
00351 "regPhySubFile: rsRegReplica error for %s. status = %d",
00352 bunDataObjInfo->objPath, status);
00353 return status;
00354 }
00355
00356 return status;
00357 }
00358
00359 int unbunPhyBunFile( rsComm_t *rsComm, char *objPath,
00360 rescInfo_t *rescInfo, char *bunFilePath, char *phyBunDir,
00361 char *dataType, int oprType, const char* resc_hier )
00362 {
00363 int status;
00364 structFileOprInp_t structFileOprInp;
00365
00366
00367
00368 std::string location;
00369 eirods::error ret = eirods::get_loc_for_hier_string( resc_hier, location );
00370 if( !ret.ok() ) {
00371 eirods::log( PASSMSG( "unbunPhyBunFile - failed in get_loc_for_hier_String", ret ) );
00372 return -1;
00373 }
00374
00375
00376 memset (&structFileOprInp, 0, sizeof (structFileOprInp_t));
00377 structFileOprInp.specColl = (specColl_t*)malloc (sizeof (specColl_t));
00378 memset (structFileOprInp.specColl, 0, sizeof (specColl_t));
00379 structFileOprInp.specColl->type = TAR_STRUCT_FILE_T;
00380 snprintf (structFileOprInp.specColl->collection, MAX_NAME_LEN,
00381 "%s.dir", objPath);
00382 rstrcpy (structFileOprInp.specColl->objPath,
00383 objPath, MAX_NAME_LEN);
00384 structFileOprInp.specColl->collClass = STRUCT_FILE_COLL;
00385 rstrcpy (structFileOprInp.specColl->resource, rescInfo->rescName,NAME_LEN);
00386 rstrcpy (structFileOprInp.specColl->phyPath, bunFilePath, MAX_NAME_LEN);
00387 rstrcpy (structFileOprInp.addr.hostAddr, location.c_str(), NAME_LEN );
00388
00389
00390 rstrcpy (structFileOprInp.specColl->cacheDir, phyBunDir, MAX_NAME_LEN);
00391
00392
00393
00394 if( dataType != NULL &&
00395 ( strstr (dataType, GZIP_TAR_DT_STR) != NULL ||
00396 strstr (dataType, BZIP2_TAR_DT_STR) != NULL ||
00397 strstr (dataType, ZIP_DT_STR) != NULL) ) {
00398 addKeyVal (&structFileOprInp.condInput, DATA_TYPE_KW, dataType);
00399 }
00400
00401 if ((oprType & PRESERVE_DIR_CONT) == 0)
00402 rmLinkedFilesInUnixDir (phyBunDir);
00403 structFileOprInp.oprType = oprType;
00404 status = rsStructFileExtract (rsComm, &structFileOprInp);
00405 if (status == SYS_DIR_IN_VAULT_NOT_EMPTY) {
00406
00407 if (chkOrphanDir (rsComm, phyBunDir, rescInfo->rescName) > 0) {
00408
00409 fileRenameInp_t fileRenameInp;
00410 bzero (&fileRenameInp, sizeof (fileRenameInp));
00411 rstrcpy (fileRenameInp.oldFileName, phyBunDir, MAX_NAME_LEN);
00412 status = renameFilePathToNewDir (rsComm, ORPHAN_DIR,
00413 &fileRenameInp, rescInfo, 1);
00414
00415 if (status >= 0) {
00416 rodsLog (LOG_NOTICE,
00417 "unbunPhyBunFile: %s has been moved to ORPHAN_DIR.stat=%d",
00418 phyBunDir, status);
00419 status = rsStructFileExtract (rsComm, &structFileOprInp);
00420 } else {
00421 rodsLog (LOG_ERROR,
00422 "unbunPhyBunFile: renameFilePathToNewDir err for %s.stat=%d",
00423 phyBunDir, status);
00424 status = SYS_DIR_IN_VAULT_NOT_EMPTY;
00425 }
00426 }
00427 }
00428 clearKeyVal (&structFileOprInp.condInput);
00429 if (status < 0) {
00430 rodsLog (LOG_ERROR,
00431 "unbunPhyBunFile: rsStructFileExtract err for %s. status = %d",
00432 objPath, status);
00433 }
00434 free (structFileOprInp.specColl);
00435
00436 return (status);
00437 }
00438
00439 int
00440 remoteUnbunAndRegPhyBunfile (rsComm_t *rsComm, dataObjInp_t *dataObjInp,
00441 rodsServerHost_t *rodsServerHost)
00442 {
00443 int status;
00444
00445 if (rodsServerHost == NULL) {
00446 rodsLog (LOG_NOTICE,
00447 "remoteUnbunAndRegPhyBunfile: Invalid rodsServerHost");
00448 return SYS_INVALID_SERVER_HOST;
00449 }
00450
00451 if ((status = svrToSvrConnect (rsComm, rodsServerHost)) < 0) {
00452 return status;
00453 }
00454
00455 status = rcUnbunAndRegPhyBunfile (rodsServerHost->conn, dataObjInp);
00456
00457 return status;
00458 }
00459
00460 int
00461 rmLinkedFilesInUnixDir (char *phyBunDir)
00462 {
00463 #ifndef USE_BOOST_FS
00464 DIR *dirPtr;
00465 struct dirent *myDirent;
00466 struct stat statbuf;
00467 #endif
00468 char subfilePath[MAX_NAME_LEN];
00469 int status;
00470 int linkCnt;
00471
00472 #ifdef USE_BOOST_FS
00473 path srcDirPath (phyBunDir);
00474 if (!exists(srcDirPath) || !is_directory(srcDirPath)) return 0;
00475 #else
00476 dirPtr = opendir (phyBunDir);
00477 if (dirPtr == NULL) return 0;
00478 #endif
00479
00480 #ifdef USE_BOOST_FS
00481 directory_iterator end_itr;
00482 for (directory_iterator itr(srcDirPath); itr != end_itr;++itr) {
00483 path p = itr->path();
00484 snprintf (subfilePath, MAX_NAME_LEN, "%s",
00485 p.c_str ());
00486 #else
00487 while ((myDirent = readdir (dirPtr)) != NULL) {
00488 if (strcmp (myDirent->d_name, ".") == 0 ||
00489 strcmp (myDirent->d_name, "..") == 0) {
00490 continue;
00491 }
00492 snprintf (subfilePath, MAX_NAME_LEN, "%s/%s",
00493 phyBunDir, myDirent->d_name);
00494 #endif
00495 #ifdef USE_BOOST_FS
00496 if (!exists (p)) {
00497 continue;
00498 }
00499
00500 if (is_regular_file(p)) {
00501 if ((linkCnt = hard_link_count (p)) >= 2) {
00502 #else
00503 status = stat (subfilePath, &statbuf);
00504
00505 if (status != 0) {
00506 continue;
00507 }
00508
00509 if ((statbuf.st_mode & S_IFREG) != 0) {
00510 if ((linkCnt = statbuf.st_nlink) >= 2) {
00511 #endif
00512
00513 unlink (subfilePath);
00514 } else {
00515 rodsLog (LOG_ERROR,
00516 "rmLinkedFilesInUnixDir: st_nlink of %s is only %d",
00517 subfilePath, linkCnt);
00518 }
00519 } else {
00520 status = rmLinkedFilesInUnixDir (subfilePath);
00521
00522 rmdir (subfilePath);
00523 }
00524 }
00525 #ifndef USE_BOOST_FS
00526 closedir (dirPtr);
00527 #endif
00528 return 0;
00529 }
00530
00531 int
00532 rmUnlinkedFilesInUnixDir (char *phyBunDir)
00533 {
00534 DIR *dirPtr;
00535 struct dirent *myDirent;
00536 struct stat statbuf;
00537 int status;
00538 char subfilePath[MAX_NAME_LEN];
00539 time_t myTime = time (0) - UNLINK_FILE_AGE;
00540
00541 dirPtr = opendir (phyBunDir);
00542 if (dirPtr == NULL) return 0;
00543 while ((myDirent = readdir (dirPtr)) != NULL) {
00544 if (strcmp (myDirent->d_name, ".") == 0 ||
00545 strcmp (myDirent->d_name, "..") == 0) {
00546 continue;
00547 }
00548 snprintf (subfilePath, MAX_NAME_LEN, "%s/%s",
00549 phyBunDir, myDirent->d_name);
00550 status = stat (subfilePath, &statbuf);
00551
00552 if (status != 0) {
00553 continue;
00554 }
00555
00556 if ((statbuf.st_mode & S_IFREG) != 0) {
00557
00558
00559 if (statbuf.st_nlink == 1 && statbuf.st_mtime > myTime)
00560 unlink (subfilePath);
00561 } else {
00562 status = rmUnlinkedFilesInUnixDir (subfilePath);
00563
00564 rmdir (subfilePath);
00565 }
00566 }
00567 closedir (dirPtr);
00568 return 0;
00569 }
00570