Index: menu.c =================================================================== --- menu.c (.../base) (Revision 935) +++ menu.c (.../hlcutter) (Revision 935) @@ -2666,7 +2666,9 @@ Add(new cMenuEditStrItem( tr("Setup.Recording$Name instant recording"), data.NameInstantRecord, sizeof(data.NameInstantRecord), tr(FileNameChars))); Add(new cMenuEditIntItem( tr("Setup.Recording$Instant rec. time (min)"), &data.InstantRecordTime, 1, MAXINSTANTRECTIME)); Add(new cMenuEditIntItem( tr("Setup.Recording$Max. video file size (MB)"), &data.MaxVideoFileSize, MINVIDEOFILESIZE, MAXVIDEOFILESIZE)); + Add(new cMenuEditIntItem( tr("Setup.Recording$Max. recording size (GB)"), &data.MaxRecordingSize, MINRECORDINGSIZE, MAXRECORDINGSIZE)); Add(new cMenuEditBoolItem(tr("Setup.Recording$Split edited files"), &data.SplitEditedFiles)); + Add(new cMenuEditBoolItem(tr("Setup.Recording$Hard Link Cutter"), &data.HardLinkCutter)); } // --- cMenuSetupReplay ------------------------------------------------------ Index: cutter.c =================================================================== --- cutter.c (.../base) (Revision 935) +++ cutter.c (.../hlcutter) (Revision 935) @@ -13,6 +13,10 @@ #include "thread.h" #include "videodir.h" + +//#define HARDLINK_TEST_ONLY + + // --- cCuttingThread -------------------------------------------------------- class cCuttingThread : public cThread { @@ -58,6 +62,64 @@ delete toIndex; } +bool MakeHardLink(cFileName *FromFileName, cFileName *ToFileName) { + char *fromName = ReadLink(FromFileName->Name()); + char *toName = ReadLink(ToFileName->Name()); + + bool ok = true; + + // Some safety checks: + struct stat buf; + if (lstat(fromName, &buf) == 0) { + if (S_ISLNK(buf.st_mode)) { + esyslog("MakeHardLink: Failed to resolve symbolic link %s", fromName); + ok = false; + } + } + else { + esyslog("MakeHardLink: lstat failed on %s", fromName); + ok = false; + } + + if (ok && lstat(toName, &buf) == 0) { + if (S_ISLNK(buf.st_mode)) { + esyslog("MakeHardLink: Failed to resolve symbolic link %s", fromName); + ok = false; + } + else if (buf.st_size != 0) { + esyslog("MakeHardLink: File %s exists and has nonzero size", toName); + ok = false; + } + } + else if (errno != ENOENT) { + esyslog("MakeHardLink: lstat failed on %s", toName); + ok = false; + } + +#ifdef HARDLINK_TEST_ONLY + if (ok) { + // Do the hard link to *.vdr_ for testing only + char *name = NULL; + asprintf(&name, "%s_",toName); + link(fromName, name); + free(name); + ok = false; // Continue with normal copy + } +#else // HARDLINK_TEST_ONLY + if (ok) { + // Clean the existing 0-byte file + ToFileName->Close(); + unlink(toName); + // Try creating the hard link + ok = (link(fromName, toName) == 0); + } +#endif // HARDLINK_TEST_ONLY + + free(fromName); + free(toName); + return ok; +} + void cCuttingThread::Action(void) { cMark *Mark = fromMarks.First(); @@ -71,6 +133,7 @@ Mark = fromMarks.Next(Mark); int FileSize = 0; int CurrentFileNumber = 0; + bool SkipThisSourceFile = false; int LastIFrame = 0; toMarks.Add(0); toMarks.Save(); @@ -88,12 +151,66 @@ // Read one frame: - if (fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) { - if (FileNumber != CurrentFileNumber) { - fromFile = fromFileName->SetOffset(FileNumber, FileOffset); - fromFile->SetReadAhead(MEGABYTE(20)); - CurrentFileNumber = FileNumber; - } + if (!fromIndex->Get(Index++, &FileNumber, &FileOffset, &PictureType, &Length)) + break; + + if (FileNumber != CurrentFileNumber) { + fromFile = fromFileName->SetOffset(FileNumber, FileOffset); + fromFile->SetReadAhead(MEGABYTE(20)); + CurrentFileNumber = FileNumber; + if (SkipThisSourceFile) { + // At end of fast forward: Always skip to next file + toFile = toFileName->NextFile(); + if (!toFile) { + error = "toFile 4"; + break; + } + FileSize = 0; + SkipThisSourceFile = false; + } + + + if (Setup.HardLinkCutter && FileOffset == 0) { + // We are at the beginning of a new source file. + // Do we need to copy the whole file? + + // if !Mark && LastMark, then we're past the last cut-out and continue to next I-frame + // if !Mark && !LastMark, then there's just a cut-in, but no cut-out + // if Mark, then we're between a cut-in and a cut-out + + uchar MarkFileNumber; + int MarkFileOffset; + // Get file number of next cut mark + if (!Mark && !LastMark + || Mark + && fromIndex->Get(Mark->position, &MarkFileNumber, &MarkFileOffset) + && (MarkFileNumber != CurrentFileNumber)) { + // The current source file will be copied completely. + // Start new output file unless we did that already + if (FileSize != 0) { + toFile = toFileName->NextFile(); + if (!toFile) { + error = "toFile 3"; + break; + } + FileSize = 0; + } + // Try to create a hard link + if (MakeHardLink(fromFileName, toFileName)) { + // Success. Skip all data transfer for this file + SkipThisSourceFile = true; + cutIn = false; + toFile = NULL; // was deleted by toFileName->Close() + } + else { + // Fallback: Re-open the file if necessary + toFile = toFileName->Open(); + } + } + } + } + + if (!SkipThisSourceFile) { if (fromFile) { int len = ReadFrame(fromFile, buffer, Length, sizeof(buffer)); if (len < 0) { @@ -110,15 +227,12 @@ break; } } - else - break; - // Write one frame: if (PictureType == I_FRAME) { // every file shall start with an I_FRAME if (LastMark) // edited version shall end before next I-frame break; - if (FileSize > MEGABYTE(Setup.MaxVideoFileSize)) { + if (!SkipThisSourceFile && FileSize > toFileName->MaxFileSize()) { toFile = toFileName->NextFile(); if (!toFile) { error = "toFile 1"; @@ -128,12 +242,12 @@ } LastIFrame = 0; - if (cutIn) { + if (!SkipThisSourceFile && cutIn) { cRemux::SetBrokenLink(buffer, Length); cutIn = false; } } - if (toFile->Write(buffer, Length) < 0) { + if (!SkipThisSourceFile && toFile->Write(buffer, Length) < 0) { error = "safe_write"; break; } @@ -168,7 +282,7 @@ } } else - LastMark = true; + LastMark = true; // After last cut-out: Write on until next I-frame, then exit } } Recordings.TouchUpdate(); Index: README-HLCUTTER =================================================================== --- README-HLCUTTER (.../base) (Revision 0) +++ README-HLCUTTER (.../hlcutter) (Revision 935) @@ -0,0 +1,116 @@ + + VDR-HLCUTTER README + + +Written by: Udo Richter +Available at: http://www.udo-richter.de/vdr/patches.html#hlcutter + http://www.udo-richter.de/vdr/patches.en.html#hlcutter +Contact: udo_richter@gmx.de + + + +About +----- + +The hard link cutter patch changes the recording editing algorithms of VDR to +use filesystem hard links to 'copy' recording files whenever possible to speed +up editing recordings noticeably. + +The patch has matured to be quite stable, at least I'm using it without issues. +Nevertheless the patch is EXPERIMENTAL and should be used with caution. The +patch is COMPLETELY UNTESTED and NOT ADAPTED for multiple /videoxx folders. +The safety checks should prevent data loss, but hard linking may fail more +often than necessary. + +While editing a recording, the patch searches for any 00x.vdr files that dont +contain editing marks and would normally be copied 1:1 unmodified to the edited +recording. In this case the current target 00x.vdr file will be aborted, and +the cutter process attempts to duplicate the source file as a hard link, so +that both files share the same disk space. If this succeeds, the editing +process fast-forwards through the duplicated file and continues normally +beginning with the next source file. If hard linking fails, the cutter process +continues with plain old copying. (but does not take up the aborted last file.) + +After editing, the un-edited recording can be deleted as usual, the hard linked +copies will continue to exist as the only remaining copy. + +To be effective, the default 'Max. video file size (MB)' should be lowered. +The patch lowers the smallest possible file size to 1mb. Since VDR only +supports up to 255 files, this would limit the recording size to 255Mb or +10 minutes, in other words: This setting is insane! + +To make sure that the 255 file limit will not be reached, the patch also +introduces "Max. recording size (GB)" with a default of 100Gb (66 hours), and +increases the file size to 2000Mb early enough, so that 100Gb-recordings will +fit into the 255 files. + +Picking the right parameters can be tricky. The smaller the file size, the +faster the editing process works. However, with a small file size, long +recordings will fall back to 2000Mb files soon, that are slow on editing again. + +Here are some examples: + +Max file size: 100Gb 100Gb 100Gb 100Gb 100Gb 100Gb 100Gb +Max recording size: 1Mb 10Mb 20Mb 30Mb 40Mb 50Mb 100Mb + +Small files: 1-203 1-204 1-205 1-206 1-207 1-209 1-214 + GBytes: 0.2 2.0 4.0 6.0 8.1 10.2 20.9 + Hours: 0.13 1.3 2.65 4 5.4 6.8 13.9 + +Big (2000mb) files: 204-255 204-255 206-255 207-255 208-255 210-255 215-255 + GBytes: 101.5 99.6 97.7 95.7 93.8 89.8 80.1 + Hours: 67 66 65 63 62 60 53 + +A recording limit of 100Gb keeps plenty of reserve without blocking too much +file numbers. And with a file size of 30-40Mb, recordings of 4-5 hours fit into +small files completely. (depends on bit rate of course) + + + +The patch must be enabled in Setup-> Recordings-> Hard Link Cutter. When +disabled, the cutter process behaves identical to VDR's default cutter. + +There's a //#define HARDLINK_TEST_ONLY in the cutter.c file that enables a +test-mode that hard-links 00x.vdr_ files only, and continues the classic +editing. The resulting 00x.vdr and 00x.vdr_ files should be identical. If you +delete the un-edited recording, dont forget to delete the *.vdr_ files too, +they will now eat real disk space. + +Note: 'du' displays the disk space of hard links only on first appearance, and +usually you will see a noticeably smaller size on the edited recording. + + +History +------- + +Version 0.1.4 + New: Dynamic increase of file size before running out of xxx.vdr files + Fix: Last edit mark is not a cout-out + Fix: Write error if link-copied file is smaller than allowed file size + Fix: Broken index/marks if cut-in is at the start of a new file + Fix: Clear dangeling pointer to free'd cUnbufferedFile, + thx to Matthias Schwarzott + +Version 0.1.0 + Initial release + + + + +Future plans +------------ + +To support multiple /videoxx folders, the hard link must be placed on the same +disk as the source file. This requires a more advanced linking strategy. + + +Since original and edited copy share disk space, free space is wrong if one of +them is moved to *.del. Free space should only count files with hard link +count = 1. This still goes wrong if all copies get deleted. + + +For more safety, the hard-linked files may be made read-only, as modifications +to one copy will affect the other copy too. (except deleting, of course) + + +SetBrokenLink may get lost on rare cases, this needs some more thoughts. Eigenschaftsänderungen: README-HLCUTTER ___________________________________________________________________ Name: svn:executable + * Index: recorder.c =================================================================== --- recorder.c (.../base) (Revision 935) +++ recorder.c (.../hlcutter) (Revision 935) @@ -85,7 +85,7 @@ bool cFileWriter::NextFile(void) { if (recordFile && pictureType == I_FRAME) { // every file shall start with an I_FRAME - if (fileSize > MEGABYTE(Setup.MaxVideoFileSize) || RunningLowOnDiskSpace()) { + if (fileSize > fileName->MaxFileSize() || RunningLowOnDiskSpace()) { recordFile = fileName->NextFile(); fileSize = 0; } Index: config.c =================================================================== --- config.c (.../base) (Revision 935) +++ config.c (.../hlcutter) (Revision 935) @@ -272,7 +272,9 @@ FontSmlSize = 18; FontFixSize = 20; MaxVideoFileSize = MAXVIDEOFILESIZE; + MaxRecordingSize = DEFAULTRECORDINGSIZE; SplitEditedFiles = 0; + HardLinkCutter = 0; MinEventTimeout = 30; MinUserInactivity = 300; NextWakeupTime = 0; @@ -442,7 +444,9 @@ else if (!strcasecmp(Name, "FontSmlSize")) FontSmlSize = atoi(Value); else if (!strcasecmp(Name, "FontFixSize")) FontFixSize = atoi(Value); else if (!strcasecmp(Name, "MaxVideoFileSize")) MaxVideoFileSize = atoi(Value); + else if (!strcasecmp(Name, "MaxRecordingSize")) MaxRecordingSize = atoi(Value); else if (!strcasecmp(Name, "SplitEditedFiles")) SplitEditedFiles = atoi(Value); + else if (!strcasecmp(Name, "HardLinkCutter")) HardLinkCutter = atoi(Value); else if (!strcasecmp(Name, "MinEventTimeout")) MinEventTimeout = atoi(Value); else if (!strcasecmp(Name, "MinUserInactivity")) MinUserInactivity = atoi(Value); else if (!strcasecmp(Name, "NextWakeupTime")) NextWakeupTime = atoi(Value); @@ -519,7 +523,9 @@ Store("FontSmlSize", FontSmlSize); Store("FontFixSize", FontFixSize); Store("MaxVideoFileSize", MaxVideoFileSize); + Store("MaxRecordingSize", MaxRecordingSize); Store("SplitEditedFiles", SplitEditedFiles); + Store("HardLinkCutter", HardLinkCutter); Store("MinEventTimeout", MinEventTimeout); Store("MinUserInactivity", MinUserInactivity); Store("NextWakeupTime", NextWakeupTime); Index: config.h =================================================================== --- config.h (.../base) (Revision 935) +++ config.h (.../hlcutter) (Revision 935) @@ -251,7 +251,9 @@ int FontSmlSize; int FontFixSize; int MaxVideoFileSize; + int MaxRecordingSize; int SplitEditedFiles; + int HardLinkCutter; int MinEventTimeout, MinUserInactivity; time_t NextWakeupTime; int MultiSpeedMode; Index: recording.c =================================================================== --- recording.c (.../base) (Revision 935) +++ recording.c (.../hlcutter) (Revision 935) @@ -1474,6 +1474,16 @@ return NULL; } +int cFileName::MaxFileSize() { + const int smallFiles = (255 * MAXVIDEOFILESIZE - 1024 * Setup.MaxRecordingSize) + / max(MAXVIDEOFILESIZE - Setup.MaxVideoFileSize, 1); + + if (fileNumber <= smallFiles) + return MEGABYTE(Setup.MaxVideoFileSize); + + return MEGABYTE(MAXVIDEOFILESIZE); +} + cUnbufferedFile *cFileName::NextFile(void) { return SetOffset(fileNumber + 1); Index: recording.h =================================================================== --- recording.h (.../base) (Revision 935) +++ recording.h (.../hlcutter) (Revision 935) @@ -189,8 +189,16 @@ // may be slightly higher because we stop recording only before the next // 'I' frame, to have a complete Group Of Pictures): #define MAXVIDEOFILESIZE 2000 // MB -#define MINVIDEOFILESIZE 100 // MB +#define MINVIDEOFILESIZE 1 // MB +#define MINRECORDINGSIZE 25 // GB +#define MAXRECORDINGSIZE 500 // GB +#define DEFAULTRECORDINGSIZE 100 // GB +// Dynamic recording size: +// Keep recording file size at Setup.MaxVideoFileSize for as long as possible, +// but switch to MAXVIDEOFILESIZE early enough, so that Setup.MaxRecordingSize +// will be reached, before recording to file 255.vdr + class cIndexFile { private: struct tIndex { int offset; uchar type; uchar number; short reserved; }; @@ -230,6 +238,8 @@ cUnbufferedFile *Open(void); void Close(void); cUnbufferedFile *SetOffset(int Number, int Offset = 0); + int MaxFileSize(); + // Dynamic file size for this file cUnbufferedFile *NextFile(void); };