Add --tail=NUM to limit the number of items to keep in memory

src/chunklist.go

@@ -48,7 +48,12 @@ func CountItems(cs []*Chunk) int {
 	if len(cs) == 0 {
 		return 0
 	}
-	return chunkSize*(len(cs)-1) + cs[len(cs)-1].count
+	if len(cs) == 1 {
+		return cs[0].count
+	}
+
+	// First chunk might not be full due to --tail=N
+	return cs[0].count + chunkSize*(len(cs)-2) + cs[len(cs)-1].count
 }
 
 // Push adds the item to the list
@@ -72,18 +77,53 @@ func (cl *ChunkList) Clear() {
 }
 
 // Snapshot returns immutable snapshot of the ChunkList
-func (cl *ChunkList) Snapshot() ([]*Chunk, int) {
+func (cl *ChunkList) Snapshot(tail int) ([]*Chunk, int, bool) {
 	cl.mutex.Lock()
 
+	changed := false
+	if tail > 0 && CountItems(cl.chunks) > tail {
+		changed = true
+		// Find the number of chunks to keep
+		numChunks := 0
+		for left, i := tail, len(cl.chunks)-1; left > 0 && i >= 0; i-- {
+			numChunks++
+			left -= cl.chunks[i].count
+		}
+
+		// Copy the chunks to keep
+		ret := make([]*Chunk, numChunks)
+		copy(ret, cl.chunks[len(cl.chunks)-numChunks:])
+
+		for left, i := tail, len(ret)-1; i >= 0; i-- {
+			chunk := ret[i]
+			if chunk.count > left {
+				newChunk := *chunk
+				newChunk.count = left
+				oldCount := chunk.count
+				for i := 0; i < left; i++ {
+					newChunk.items[i] = chunk.items[oldCount-left+i]
+				}
+				ret[i] = &newChunk
+				break
+			}
+			left -= chunk.count
+		}
+		cl.chunks = ret
+	}
+
 	ret := make([]*Chunk, len(cl.chunks))
 	copy(ret, cl.chunks)
 
-	// Duplicate the last chunk
+	// Duplicate the first and the last chunk
 	if cnt := len(ret); cnt > 0 {
+		if tail > 0 && cnt > 1 {
+			newChunk := *ret[0]
+			ret[0] = &newChunk
+		}
 		newChunk := *ret[cnt-1]
 		ret[cnt-1] = &newChunk
 	}
 
 	cl.mutex.Unlock()
-	return ret, CountItems(ret)
+	return ret, CountItems(ret), changed
 }