Brief description of class still missing. More...

#include <Process.h>

Inheritance diagram for Process:

Signals
void	readyToPick (SubSignalPool *subPool, QString key, TimeRange recRange)
Public Member Functions
bool	failed () const
	Process ()
bool	setArguments (int &argc, char **argv)
	~Process ()
Static Public Attributes
static const QString	defaultDataDir = "../data"
static const QString	defaultDestDir = "../sac"
static const QString	defaultStationFile = "stations.txt"
static const QString	defaultTimeDir = "../timedrift"
static const QString	defaultTitanFormatFile = "/kephrenrtu/titanUncorrectedT0.ascfmt"
static const QString	defaultWorkDir = "."
Protected Member Functions
virtual void	run ()

Detailed Description

Brief description of class still missing.

Full description of class still missing

Constructor & Destructor Documentation

Process::Process ( )

Description of constructor still missing

References QGpCoreTools::SharedObject::addReference(), defaultDataDir, defaultDestDir, defaultStationFile, defaultTimeDir, defaultTitanFormatFile, defaultWorkDir, readyToPick(), and TRACE.

{
  TRACE;
  _dataDir=defaultDataDir;
  _timeDir=defaultTimeDir;
  _stationFile=defaultStationFile;
  _destDir=defaultDestDir;
  _workDir=defaultWorkDir;
  _titanFormatFile=defaultTitanFormatFile;
  _samplingFrequency=100.0;
  _pickOnly=false;
  _failed=false;
  connect(this, SIGNAL(readyToPick(SubSignalPool *, QString, TimeRange)),
          this, SLOT(pick(SubSignalPool *, QString, TimeRange)), Qt::QueuedConnection);
  _pickToolCount=0;
  // Special TITAN format
  _titanFormat.addReference(); // Prevent its deletion
}

Process::~Process ( )

Description of destructor still missing

References QGpCoreTools::SharedObject::removeReference(), and TRACE.

{
  TRACE;
  _titanFormat.removeReference(); // Allow its deletion
}

Member Function Documentation

bool Process::failed ( ) const [inline]

Referenced by main().

{return _failed;}

void Process::readyToPick	(	SubSignalPool *	subPool,
		QString	key,
		TimeRange	recRange
	)		`[signal]`

Referenced by Process(), and run().

void Process::run ( ) [protected, virtual]

{
  TRACE;
  QStringList infoFilter, ascFilter, ftpFilter, tmpWorkFilter;
  infoFilter.append("*.info");
  ascFilter.append("*.ASC");
  ftpFilter.append("*.*.*");
  tmpWorkFilter.append("tmp_work_*");
  SignalFileFormat fmt(SignalFileFormat::Ascii, &_titanFormat);
  QStringList tmpWorkDirs=_workDir.entryList(tmpWorkFilter, QDir::NoDotAndDotDot | QDir::Dirs);
  foreach(QString wd, tmpWorkDirs) {
    QDir d(_workDir);
    d.cd(wd);
    QStringList tmpFiles=d.entryList(QDir::Files);
    foreach(QString f, tmpFiles) {
      d.remove(f);
    }
    if(!_workDir.rmdir(wd)) {
      App::stream() << tr("Cannot remove directory %1").arg(wd) << endl;
      return;
    }
  }
  App::stream() << tr("Start loop ...") << endl;
  QStringList * currentStations=0;
  int tempDirIndex=100000;
  for(ScheduleList::iterator itSched=_schedule.begin();itSched!=_schedule.end();itSched++) {
    App::stream() << tr("%1/%2").arg(itSched->site).arg(itSched->array) << endl;
    QDir siteDestDir(_destDir);
    siteDestDir.cd(itSched->site);
    siteDestDir.cd(itSched->array);
    DateTime recStart, recEnd;
    recStart.setDateTime(itSched->start);
    recEnd.setDateTime(itSched->end);
    TimeRange recRange(_timeReference.secondsTo(recStart), _timeReference.secondsTo(recEnd));
    if(itSched->array.startsWith("HV")) {
      currentStations=&_hvStations;
      _hvStations.clear();
      _hvStations.append(itSched->info);
    } else {
      currentStations=&_arrayStations;
    }
    for(QStringList::iterator itStation=currentStations->begin(); itStation!=currentStations->end(); itStation++) {
      Curve<Point2D>& statUncorr2Corr=_timeUncorr2Corr[*itStation];
      // Create a new temporary working directory
      QString tmpWorkDirName="tmp_work_"+*itStation+"_"+QString::number(tempDirIndex);
      _workDir.mkdir(tmpWorkDirName);
      QDir tmpWorkDir(_workDir);
      tmpWorkDir.cd(tmpWorkDirName);
      tempDirIndex++;

      QProcess p;
      p.setWorkingDirectory(tmpWorkDir.absolutePath());
      QStringList args;
      args.append("-ts"); // Bug in titan... if this option is not there, timing is not correct
      args.append("86400"); // Ask for the maximum possible length
      args.append("-asc"); // Ask for ASCII files
      args.append("-f"); // Set input as files
      for(QDate day=itSched->start.date(); day<=itSched->end.date();day=day.addDays(1)) {
        QDir d(_dataDir);
        d.cd(*itStation);
        d.cd("ftp");
        d.cd(*itStation);
        d.cd(day.toString("yyyy.MM.dd"));
        QStringList ftpFiles=d.entryList(ftpFilter, QDir::Files | QDir::NoDotAndDotDot);
        foreach(QString f, ftpFiles) {
          args.append(d.absoluteFilePath(f));
        }
      }
      App::stream() << tr("rtitan2 -ts 86400 -asc -f ... ");
      p.start("rtitan2", args);
      while(!p.waitForFinished()) {}
      // Remove *.info files
      QStringList infoFiles=tmpWorkDir.entryList(infoFilter, QDir::Files | QDir::NoDotAndDotDot);
      if(infoFiles.isEmpty()) {
        App::stream() << tr("error in rtitan2") << endl;
      } else {
        App::stream() << tr("ok") << endl;
      }
      foreach(QString f, infoFiles) {
        tmpWorkDir.remove(f);
      }
      QString manualLimitsKey;
      QStringList ascFiles=tmpWorkDir.entryList(ascFilter, QDir::Files | QDir::NoDotAndDotDot);
      foreach(QString signalFile, ascFiles) {
        SubSignalPool subPool=geopsyCore->load(tmpWorkDir.absoluteFilePath(signalFile), fmt);
        if(subPool.count()!=1) {
          _failed=true;
          return;
        }
        Signal * sig=subPool.first();
        subPool.remove(sig);
        if(sig->name()!=*itStation || sig->component()==Signal::UndefinedComponent) {
          App::stream() << tr("skip (bad name or component)") << endl;
          tmpWorkDir.remove(signalFile);
          geopsyCore->currentDB()->removeFile(sig->file());
          continue;
        }
        if(fabs(sig->samplingFrequency()-_samplingFrequency)>1e-10) {
          App::stream() << tr("skip (sampling at %1 Hz instead of %2 Hz)").arg(sig->samplingFrequency()).arg(_samplingFrequency) << endl;
          tmpWorkDir.remove(signalFile);
          geopsyCore->currentDB()->removeFile(sig->file());
          continue;
        }
        if(sig->timePick("adcDelay")!=_adcDelay) {
          App::stream() << tr("adc delay=%1 (expected %2)").arg(sig->timePick("adcDelay")).arg(_adcDelay) << endl;
          tmpWorkDir.remove(signalFile);
          geopsyCore->currentDB()->removeFile(sig->file());
          _failed=true;
          return;
        }
        if(sig->timePick("filterDelay")!=_filterDelay) {
          App::stream() << tr("filter delay=%1 (expected %2)").arg(sig->timePick("filterDelay")).arg(_filterDelay) << endl;
          tmpWorkDir.remove(signalFile);
          geopsyCore->currentDB()->removeFile(sig->file());
          _failed=true;
          return;
        }
        // Set time reference to global timeReference, t0 are uncorrected
        DateTime t0Sig;
        t0Sig.setDateTime(sig->timeReference());
        t0Sig.addSeconds(sig->t0());
        sig->setTimeReference(_timeReference.dateTime());
        sig->setT0(_timeReference.secondsTo(t0Sig));

        // Cut signal with manual limits
        _manualLimitsMutex.lock();
        manualLimitsKey=itSched->site+":"+itSched->array+":"+*itStation;
        QMap<QString, SparseTimeRange>::iterator it=_manualLimits.find(manualLimitsKey);
        if(_pickOnly) {
          if(it!=_manualLimits.end()) {
            const QVector<TimeRange>& rList=it.value().ranges();
            int i=1;
            for(QVector<TimeRange>::const_iterator itr=rList.begin(); itr!=rList.end(); itr++, i++) {
              sig->setTimePick(QString("%1_Begin").arg(i), itr->start());
              sig->setTimePick(QString("%1___End").arg(i), itr->end());
            }
          }
          _manualLimitsMutex.unlock();
          // Force load of samples
          sig->maximumAmplitude();
          _pickSubPool.addSignal(sig);
          continue;
        }

        if(it==_manualLimits.end()) {
          geopsyCore->currentDB()->removeFile(sig->file());
          _manualLimitsMutex.unlock();
          App::stream() << tr("No manual limits found for %1").arg(manualLimitsKey) << endl;
          _failed=true;
          return;
        }
        _manualLimitsMutex.unlock();

        const QVector<TimeRange>& manualCutRanges=it.value().ranges();
        for(QVector<TimeRange>::const_iterator itr=manualCutRanges.begin(); itr!=manualCutRanges.end(); itr++) {
          TimeRange manualCutRange=*itr;
          Signal * cutsig=sig->cut(manualCutRange);
          if(!cutsig) {
            App::stream() << tr("No signal found.") << endl;
            continue;
          }

          // High-pass filter
          FilterParameters filterParam;
          filterParam.setMinimumFrequency(0.05);
          filterParam.setBand(FilterParameters::HighPass);
          filterParam.setMethod(FilterParameters::Convolution);
          filterParam.convolutionWindow().setWindow(TaperParameters::Tukey);
          filterParam.setWidth(0.5);
          cutsig->filter(filterParam);

          // Do not include the first and last 30 seconds (filter border effects)
          Signal * cutsig30=cutsig->cut(TimeRange(cutsig->t0()+30, cutsig->endTime()-30));
          geopsyCore->currentDB()->removeSignal(cutsig);
          if(!cutsig30) {
            App::stream() << tr("Too short signal.") << endl;
            geopsyCore->currentDB()->removeSignal(cutsig30);
            continue;
          }
          cutsig=cutsig30;

          // Cut signal in blocks of 15 minutes
          double tUncorr=cutsig->t0();
          while(tUncorr<cutsig->endTime()) {
            /*DateTime currentTime(_timeReference);
            currentTime.addSeconds(tUncorr);
            App::stream() << "Current time: "
                          << currentTime.toString("yyyy-MM-dd hh:mm:ssz") << endl;*/
            double tCorr=statUncorr2Corr.at(tUncorr).y();
            double tCorrEnd=(floor(tCorr/900.0)+1.0)*900;
            if(tCorrEnd-tCorr<60) tCorrEnd+=900; // Avoid blocks of less than 1 minute
            TimeRange blockRecRange=TimeRange(tCorr, tCorrEnd).intersection(recRange);
            if(blockRecRange.lengthSeconds()==0) {
              tUncorr+=cutsig->deltaT()*floor((tCorrEnd-tCorr)/cutsig->deltaT());
              continue;
            }
            if(blockRecRange.start()>tCorr) { // Changing t0, we must keep a round number of samples
              double dt=ceil((blockRecRange.start()-tCorr)/cutsig->deltaT())*cutsig->deltaT();
              tCorr+=dt;
              tUncorr+=dt;
            }
            if(tCorrEnd-tCorr<60) { // Avoid blocks of less than 1 minute
              tCorrEnd+=900;
              // Update blockRecRange...
              blockRecRange=TimeRange(tCorr, tCorrEnd).intersection(recRange);
            }
            if(blockRecRange.end()<tCorrEnd) { // Changing end time
              tCorrEnd=blockRecRange.end();
            }
            if(tCorrEnd-tCorr<60.0) { // Less than a minute, drop it
              tUncorr+=900.0;
              continue;
            }

            TimeRange blockRange(tUncorr, tUncorr+(tCorrEnd-tCorr));

            DateTime tCorrTime(_timeReference);
            tCorrTime.addSeconds(tCorr);
            DateTime tCorrEndTime(_timeReference);
            tCorrEndTime.addSeconds(tCorrEnd);
            App::stream() << "t (corr)         ["
                          << tCorrTime.toString("yyyy-MM-dd hh:mm:ssz") << ", "
                          << tCorrEndTime.toString("yyyy-MM-dd hh:mm:ssz") << "]" << endl;
            DateTime tUncorrTime(_timeReference);
            tUncorrTime.addSeconds(blockRange.start());
            DateTime tUncorrEndTime(_timeReference);
            tUncorrEndTime.addSeconds(blockRange.end());
            App::stream() << "t (uncorr)       ["
                          << tUncorrTime.toString("yyyy-MM-dd hh:mm:ssz") << ", "
                          << tUncorrEndTime.toString("yyyy-MM-dd hh:mm:ssz") << "]" << endl;
            DateTime sUncorrTime(_timeReference);
            sUncorrTime.addSeconds(cutsig->t0());
            DateTime sUncorrEndTime(_timeReference);
            sUncorrEndTime.addSeconds(cutsig->endTime());
            App::stream() << "sig (uncorr)     ["
                          << sUncorrTime.toString("yyyy-MM-dd hh:mm:ssz") << ", "
                          << sUncorrEndTime.toString("yyyy-MM-dd hh:mm:ssz") << "]" << endl;

            // Cut signal
            Signal * cutsig15=cutsig->cut(blockRange);
            if(!cutsig15) {
              break;
            }

            DateTime scUncorrTime(_timeReference);
            scUncorrTime.addSeconds(cutsig15->timeRange().range().start());
            DateTime scUncorrEndTime(_timeReference);
            scUncorrEndTime.addSeconds(cutsig15->timeRange().range().end());
            App::stream() << "cut sig (uncorr) ["
                          << scUncorrTime.toString("yyyy-MM-dd hh:mm:ssz") << ", "
                          << scUncorrEndTime.toString("yyyy-MM-dd hh:mm:ssz") << "]" << endl;

            // Go to next time block (step a round number of samples)
            tUncorr+=ceil((tCorrEnd-tCorr)/cutsig15->deltaT())*cutsig15->deltaT();

            // Shift and correct time
            double tRounded=floor(tCorr/cutsig->deltaT())*cutsig->deltaT();
            cutsig15->shift(tRounded-tCorr);
            cutsig15->setT0(tRounded);
            App::stream() << "Shift by " << (tRounded-tCorr) << " s" << endl;

            // Remove first and last samples (2, to force a hole in all cases)
            Signal * finalSig=new Signal(*cutsig15);
            finalSig->setT0(tRounded+2*cutsig->deltaT());
            finalSig->setNSamples(cutsig15->nSamples()-4);
            finalSig->copySamplesFrom(cutsig15, finalSig->timeRange().range());
            // WAU49 had a sensor with reversed connections (not always)
            if(*itStation=="WAU49" && itSched->info.toInt()==-1) {
              finalSig->multiply(-1);
            }
            // Save shifted signal
            SubSignalPool savePool;
            savePool.addSignal(finalSig);
            DateTime timeBlock(_timeReference);
            timeBlock.addSeconds(finalSig->t0());
            DateTime endTimeBlock(_timeReference);
            endTimeBlock.addSeconds(finalSig->endTime());
            App::stream() << timeBlock.toString("yyyy-MM-dd hh:mm:ssz") << " ---> "
                          << endTimeBlock.toString("yyyy-MM-dd hh:mm:ssz") << endl;
            savePool.save(siteDestDir.absoluteFilePath(*itStation+"_"+
                                                       timeBlock.toString("yyyyMMdd-hhmm")+"_"+
                                                       Signal::componentLetter(cutsig->component())+".sac"),
                                                       false, SignalFileFormat::SacBigEndian);
            savePool.remove(finalSig);

            geopsyCore->currentDB()->removeSignal(cutsig15);
            geopsyCore->currentDB()->removeSignal(finalSig);
          }
          geopsyCore->currentDB()->removeSignal(cutsig);
        }
        tmpWorkDir.remove(signalFile);
        geopsyCore->currentDB()->removeFile(sig->file());
        geopsyCore->clear();
      }
      if(_pickOnly) {
        _pickToolMutex.lock();
        while(_pickToolCount>=3) {
          _pickToolMutex.unlock();
          msleep(1000);
          _pickToolMutex.lock();
          _removeMutex.lock();
          foreach(Signal * sig, _toRemove) {
            QFile(sig->file()->name()).remove();
            geopsyCore->currentDB()->removeFile(sig->file());
          }
          _toRemove.clear();
          _removeMutex.unlock();
        }
        _pickToolCount++;
        _pickToolMutex.unlock();
        emit readyToPick(new SubSignalPool(_pickSubPool), manualLimitsKey, recRange);
        _pickSubPool.removeAll();
      }
    }
  }
  if(_pickOnly) {
    _pickToolMutex.lock();
    while(_pickToolCount>0) {
      _pickToolMutex.unlock();
      msleep(1000);
      _pickToolMutex.lock();
      _removeMutex.lock();
      foreach(Signal * sig, _toRemove) {
        QFile(sig->file()->name()).remove();
        geopsyCore->currentDB()->removeFile(sig->file());
      }
      _toRemove.clear();
      _removeMutex.unlock();
    }
  }
}

bool Process::setArguments	(	int &	argc,
		char **	argv
	)

References QGpCoreTools::endl(), QGpCoreTools::DateTime::fromString(), Schedule::read(), QGpCoreTools::tr(), and QGpCoreTools::XMLHeader::xml_restoreFile().

Referenced by main().

{
  // Check arguments
  int i, j=1;
  for(i=1; i<argc; i++) {
    QByteArray arg=argv[i];
    if(arg[0]=='-') {
      if(arg=="-data") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _dataDir.setPath(argv[i]);
      } else if(arg=="-time") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _timeDir.setPath(argv[i]);
      } else if(arg=="-stations") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _stationFile=argv[i];
      } else if(arg=="-dest") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _destDir.setPath(argv[i]);
      } else if(arg=="-fmt") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _titanFormatFile=argv[i];
      } else if(arg=="-time-ref") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _timeReference.fromString(argv[i], "yyyy-MM-dd");
      } else if(arg=="-pick-only") {
        _pickOnly=true;
      } else if(arg=="-sampling-frequency") {
        CoreApplication::checkOptionArg(i, argc, argv);
        _samplingFrequency=atof(argv[i]);
      } else {
        App::stream() << tr("kephrenrtu: bad option %1, see -help").arg(argv[i]) << endl;
        return false;
      }
    } else {
      argv[j++]=argv[i];
    }
  }
  if(j < argc) {
    argv[j]=0;
    argc=j;
  }

  if(_samplingFrequency==100.0) {
    _adcDelay=0.000156;
    _filterDelay=0.507969;
  } else if(_samplingFrequency==250.0) {
    _adcDelay=0.000156;
    _filterDelay=0.224469;
  } else {
    App::stream() << tr("Unsupported sampling frequency (%1 Hz)").arg(_samplingFrequency) << endl;
    return false;
  }

  // Read manual limits
  App::stream() << tr("Read manual limits ...") << endl;
  _manualLimitsFileName=_destDir.absoluteFilePath("manuallimits");
  loadManualLimits();
  // Read _schedule
  App::stream() << tr("Read _schedule ...") << endl;
  _schedule=Schedule::read(_dataDir.absoluteFilePath("schedule.txt"));
  App::stream() << tr("  found %1 entries").arg(_schedule.count()) << endl;
  // Read station list
  App::stream() << tr("Read stations ...") << endl;
  _arrayStations=readStations();
  if(_arrayStations.isEmpty()) return 2;
  App::stream() << tr("  found %1 stations").arg(_arrayStations.count()) << endl;
  // Build time corrections
  App::stream() << tr("Read time corrections ...") << endl;
  if(!loadTimeCorrections()) return 2;
  // TODO: remove -fmt option and build it internally
  if(_titanFormatFile.isEmpty()) {
     App::stream() << tr("Missing titan format file") << endl;
     return false;
  }
  XMLHeader hdr(&_titanFormat);
  if(hdr.xml_restoreFile(_titanFormatFile)!=XMLClass::NoError) {
    App::stream() << tr("Error opening file %1").arg(_titanFormatFile) << endl;
    return false;
  }

  return true;
}

Member Data Documentation

const QString Process::defaultDataDir = "../data" [static]

Referenced by Process().

const QString Process::defaultDestDir = "../sac" [static]

Referenced by Process().

const QString Process::defaultStationFile = "stations.txt" [static]

Referenced by Process().

const QString Process::defaultTimeDir = "../timedrift" [static]

Referenced by Process().

const QString Process::defaultTitanFormatFile = "/kephrenrtu/titanUncorrectedT0.ascfmt" [static]

Referenced by Process().

const QString Process::defaultWorkDir = "." [static]

Referenced by Process().

The documentation for this class was generated from the following files:

kephrenrtu/Process.h
kephrenrtu/Process.cpp

Signals

Public Member Functions

Static Public Attributes

Protected Member Functions

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation