Index: README-HLCUTTER =================================================================== --- README-HLCUTTER (.../base) (Revision 0) +++ README-HLCUTTER (.../hlcutter) (Revision 894) @@ -0,0 +1,79 @@ + + VDR-HLCUTTER README + + +Written by: Udo Richter +Available at: http://www.udo-richter.de/vdr/patches.html#hlcutter + http://www.udo-richter.de/vdr/patches.en.html#hlcutter +Contact: udo_richter@gmx.de + + + +About +----- + +The hard link cutter patch changes the recording editing algorithms of VDR to +use filesystem hard links to 'copy' recording files whenever possible to speed +up editing recordings noticeably. + +Though the patch has been working for a week for me very well, it is HIGHLY +EXPERIMENTAL and should be used with caution. The patch is COMPLETELY UNTESTED +and NOT ADAPTED for multiple /videoxx folders. The safety checks should prevent +data loss, but hard linking may fail more often than necessary. + + +While editing a recording, the patch searches for any 00x.vdr files that dont +contain editing marks and would normally be copied 1:1 unmodified to the edited +recording. In this case the current target 00x.vdr file will be aborted, and +the cutter process attempts to duplicate the source file as a hard link, so +that both files share the same disk space. If this succeeds, the editing +process fast-forwards through the duplicated file and continues normally +beginning with the next source file. If hard linking fails, the cutter process +continues with plain old copying. (but does not take up the aborted last file.) + +After editing, the un-edited recording can be deleted as usual, the hard linked +copies will continue to exist as the only remaining copy. + +To be effective, the default 'Max. video file size (MB)' should be lowered. +The patch lowers the smallest possible file size to 10mb, though this limits +the recording length to 10*255 Mb (~100 minutes). A value of 100mb is a good +compromise between speed improvement and recording length (~16 hours). + +The patch must be enabled in Setup-> Recordings-> Hard Link Cutter. When +disabled, the cutter process behaves identical to VDR's default cutter. + +There's a //#define HARDLINK_TEST_ONLY in the cutter.c file that enables a +test-mode that hard-links 00x.vdr_ files only, and continues the classic +editing. The resulting 00x.vdr and 00x.vdr_ files should be identical. If you +delete the un-edited recording, dont forget to delete the *.vdr_ files too, +they will now eat real disk space. + +Note: 'du' displays the disk space of hard links only on first appearance, and +usually you will see a noticeably smaller size on the edited recording. + + +Future plans +------------ + +To solve the file size vs. recording size conflict, dynamic file sizes could be +implemented, so that a recording starts with small file sizes, and increases +the file size at some point to ensure enough space for huge recordings before +255.vdr is reached. For example, using 32Mb up to 192.vdr and 2000mb from +193.vdr on will give a total of 128 Gb or 84 hours, while using small files for +up to 4 hours. + + +To support multiple /videoxx folders, the hard link must be placed on the same +disk as the source file. This requires a more advanced linking strategy. + + +Since original and edited copy share disk space, free space is wrong if one of +them is moved to *.del. Free space should only count files with hard link +count = 1. This still goes wrong if all copies get deleted. + + +For more safety, the hard-linked files may be made read-only, as modifications +to one copy will affect the other copy too. (except deleting, of course) + + +SetBrokenLink may get lost on rare cases, this needs some more thoughts. Index: menu.c =================================================================== --- menu.c (.../base) (Revision 894) +++ menu.c (.../hlcutter) (Revision 894) @@ -2622,6 +2622,7 @@ Add(new cMenuEditIntItem( tr("Setup.Recording$Instant rec. time (min)"), &data.InstantRecordTime, 1, MAXINSTANTRECTIME)); Add(new cMenuEditIntItem( tr("Setup.Recording$Max. video file size (MB)"), &data.MaxVideoFileSize, MINVIDEOFILESIZE, MAXVIDEOFILESIZE)); Add(new cMenuEditBoolItem(tr("Setup.Recording$Split edited files"), &data.SplitEditedFiles)); + Add(new cMenuEditBoolItem(tr("Setup.Recording$Hard Link Cutter"), &data.HardLinkCutter)); } // --- cMenuSetupReplay ------------------------------------------------------ Index: cutter.c =================================================================== --- cutter.c (.../base) (Revision 894) +++ cutter.c (.../hlcutter) (Revision 894) @@ -13,6 +13,10 @@ #include "thread.h" #include "videodir.h" + +//#define HARDLINK_TEST_ONLY + + // --- cCuttingThread -------------------------------------------------------- class cCuttingThread : public cThread { @@ -58,6 +62,64 @@ delete toIndex; } +bool MakeHardLink(cFileName *FromFileName, cFileName *ToFileName) { + char *fromName = ReadLink(FromFileName->Name()); + char *toName = ReadLink(ToFileName->Name()); + + bool ok = true; + + // Some safety checks: + struct stat buf; + if (lstat(fromName, &buf) == 0) { + if (S_ISLNK(buf.st_mode)) { + esyslog("MakeHardLink: Failed to resolve symbolic link %s", fromName); + ok = false; + } + } + else { + esyslog("MakeHardLink: lstat failed on %s", fromName); + ok = false; + } + + if (ok && lstat(toName, &buf) == 0) { + if (S_ISLNK(buf.st_mode)) { + esyslog("MakeHardLink: Failed to resolve symbolic link %s", fromName); + ok = false; + } + else if (buf.st_size != 0) { + esyslog("MakeHardLink: File %s exists and has nonzero size", toName); + ok = false; + } + } + else if (errno != ENOENT) { + esyslog("MakeHardLink: lstat failed on %s", toName); + ok = false; + } + +#ifdef HARDLINK_TEST_ONLY + if (ok) { + // Do the hard link to *.vdr_ for testing only + char *name = NULL; + asprintf(&name, "%s_",toName); + link(fromName, name); + free(name); + ok = false; // Continue with normal copy + } +#else // HARDLINK_TEST_ONLY + if (ok) { + // Clean the existing 0-byte file + ToFileName->Close(); + unlink(toName); + // Try creating the hard link + ok = (link(fromName, toName) == 0); + } +#endif // HARDLINK_TEST_ONLY + + free(fromName); + free(toName); + return ok; +} + void cCuttingThread::Action(void) { cMark *Mark = fromMarks.First(); @@ -71,6 +133,7 @@ Mark = fromMarks.Next(Mark); int FileSize = 0; int CurrentFileNumber = 0; + bool SkipThisSourceFile = false; int LastIFrame = 0; toMarks.Add(0); toMarks.Save(); @@ -88,12 +151,49 @@ // Read one frame: - if (fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) { - if (FileNumber != CurrentFileNumber) { - fromFile = fromFileName->SetOffset(FileNumber, FileOffset); - fromFile->SetReadAhead(MEGABYTE(20)); - CurrentFileNumber = FileNumber; - } + if (!fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) + break; + + if (FileNumber != CurrentFileNumber) { + fromFile = fromFileName->SetOffset(FileNumber, FileOffset); + fromFile->SetReadAhead(MEGABYTE(20)); + CurrentFileNumber = FileNumber; + SkipThisSourceFile = false; + + if (Setup.HardLinkCutter && FileOffset == 0) { + // We are at the beginning of a new source file. + // Do we need to copy the whole file? + + uchar MarkFileNumber; + int MarkFileOffset; + // Get file number of next cut mark + if (fromIndex->Get(Mark->position, &MarkFileNumber, &MarkFileOffset) + && (MarkFileNumber != CurrentFileNumber)) { + // The current source file will be copied completely. + // Start new output file unless we did that already + if (FileSize != 0) { + toFile = toFileName->NextFile(); + if (!toFile) { + error = "toFile 3"; + break; + } + FileSize = 0; + } + // Try to create a hard link + if (MakeHardLink(fromFileName, toFileName)) { + // Success. Skip all data transfer for this file + SkipThisSourceFile = true; + cutIn = false; + } + else { + // Fallback: Re-open the file + toFile = toFileName->Open(); + } + } + } + } + + if (!SkipThisSourceFile) { if (fromFile) { int len = ReadFrame(fromFile, buffer, Length, sizeof(buffer)); if (len < 0) { @@ -109,34 +209,33 @@ error = "fromFile"; break; } - } - else - break; + + // Write one frame: - // Write one frame: - - if (PictureType == I_FRAME) { // every file shall start with an I_FRAME - if (LastMark) // edited version shall end before next I-frame - break; - if (FileSize > MEGABYTE(Setup.MaxVideoFileSize)) { - toFile = toFileName->NextFile(); - if (!toFile) { - error = "toFile 1"; + if (PictureType == I_FRAME) { // every file shall start with an I_FRAME + if (LastMark) // edited version shall end before next I-frame break; + if (FileSize > MEGABYTE(Setup.MaxVideoFileSize)) { + toFile = toFileName->NextFile(); + if (!toFile) { + error = "toFile 1"; + break; + } + FileSize = 0; } - FileSize = 0; + LastIFrame = 0; + + if (cutIn) { + cRemux::SetBrokenLink(buffer, Length); + cutIn = false; + } } - LastIFrame = 0; - - if (cutIn) { - cRemux::SetBrokenLink(buffer, Length); - cutIn = false; + if (toFile->Write(buffer, Length) < 0) { + error = "safe_write"; + break; } } - if (toFile->Write(buffer, Length) < 0) { - error = "safe_write"; - break; - } + if (!toIndex->Write(PictureType, toFileName->Number(), FileSize)) { error = "toIndex"; break; Index: config.c =================================================================== --- config.c (.../base) (Revision 894) +++ config.c (.../hlcutter) (Revision 894) @@ -266,6 +266,7 @@ UseSmallFont = 1; MaxVideoFileSize = MAXVIDEOFILESIZE; SplitEditedFiles = 0; + HardLinkCutter = 0; MinEventTimeout = 30; MinUserInactivity = 300; NextWakeupTime = 0; @@ -429,6 +430,7 @@ else if (!strcasecmp(Name, "UseSmallFont")) UseSmallFont = atoi(Value); else if (!strcasecmp(Name, "MaxVideoFileSize")) MaxVideoFileSize = atoi(Value); else if (!strcasecmp(Name, "SplitEditedFiles")) SplitEditedFiles = atoi(Value); + else if (!strcasecmp(Name, "HardLinkCutter")) HardLinkCutter = atoi(Value); else if (!strcasecmp(Name, "MinEventTimeout")) MinEventTimeout = atoi(Value); else if (!strcasecmp(Name, "MinUserInactivity")) MinUserInactivity = atoi(Value); else if (!strcasecmp(Name, "NextWakeupTime")) NextWakeupTime = atoi(Value); @@ -499,6 +501,7 @@ Store("UseSmallFont", UseSmallFont); Store("MaxVideoFileSize", MaxVideoFileSize); Store("SplitEditedFiles", SplitEditedFiles); + Store("HardLinkCutter", HardLinkCutter); Store("MinEventTimeout", MinEventTimeout); Store("MinUserInactivity", MinUserInactivity); Store("NextWakeupTime", NextWakeupTime); Index: config.h =================================================================== --- config.h (.../base) (Revision 894) +++ config.h (.../hlcutter) (Revision 894) @@ -244,6 +244,7 @@ int UseSmallFont; int MaxVideoFileSize; int SplitEditedFiles; + int HardLinkCutter; int MinEventTimeout, MinUserInactivity; time_t NextWakeupTime; int MultiSpeedMode; Index: recording.h =================================================================== --- recording.h (.../base) (Revision 894) +++ recording.h (.../hlcutter) (Revision 894) @@ -187,7 +187,7 @@ // may be slightly higher because we stop recording only before the next // 'I' frame, to have a complete Group Of Pictures): #define MAXVIDEOFILESIZE 2000 // MB -#define MINVIDEOFILESIZE 100 // MB +#define MINVIDEOFILESIZE 10 // MB class cIndexFile { private: