This commit is contained in:
risk danger olson 2016-11-22 09:42:22 -07:00
parent cf908f61ae
commit 7c596b970c
9 changed files with 10 additions and 415 deletions

2
debian/rules vendored

@ -13,7 +13,7 @@ endif
BUILD_DIR := obj-$(DEB_HOST_GNU_TYPE)
export DH_GOPKG := github.com/git-lfs/git-lfs
# DH_GOLANG_EXCLUDES typically incorporates vendor exclusions from script/test
export DH_GOLANG_EXCLUDES := test github.com/olekukonko/ts/* github.com/xeipuuv/* github.com/technoweenie/go-contentaddressable/* github.com/spf13/cobra/* github.com/kr/* github.com/pkg/errors
export DH_GOLANG_EXCLUDES := test github.com/olekukonko/ts/* github.com/xeipuuv/* github.com/spf13/cobra/* github.com/kr/* github.com/pkg/errors
export PATH := $(CURDIR)/$(BUILD_DIR)/bin:$(PATH)
# by-default, dh_golang only copies *.go and other source - this upsets a bunch of vendor test routines

14
glide.lock generated

@ -1,5 +1,5 @@
hash: 8200892207101d0f4b474cd994ebb3da744f5e27037e11eaee1db6d8c2b0e559
updated: 2016-08-17T11:25:21.691448231-06:00
hash: f2f29e58b092d821d790461bd12a16239d6b8ca15db6c092fd884d82d3ef2010
updated: 2016-11-22T09:41:32.528134176-07:00
imports:
- name: github.com/bgentry/go-netrc
version: 9fd32a8b3d3d3f9d43c341bfe098430e07609480
@ -29,8 +29,8 @@ imports:
version: 6cb3b85ef5a0efef77caef88363ec4d4b5c0976d
subpackages:
- assert
- name: github.com/technoweenie/go-contentaddressable
version: 38171def3cd15e3b76eb156219b3d48704643899
- mock
- require
- name: github.com/ThomsonReutersEikon/go-ntlm
version: b00ec39bbdd04f845950f4dbb4fd0a2c3155e830
subpackages:
@ -44,10 +44,12 @@ imports:
version: d5336c75940ef31c9ceeb0ae64cf92944bccb4ee
testImports:
- name: github.com/davecgh/go-spew
version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d
version: 346938d642f2ec3594ed81d874461961cd0faa76
subpackages:
- spew
- name: github.com/pmezard/go-difflib
version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
version: 792786c7400a136282c1664665ae0a8db921c6c2
subpackages:
- difflib
- name: github.com/stretchr/objx
version: 1a9d0bb9f541897e62256577b352fdbc1fb4fd94

@ -28,8 +28,6 @@ import:
version: b00ec39bbdd04f845950f4dbb4fd0a2c3155e830
subpackages:
- ntlm
- package: github.com/technoweenie/go-contentaddressable
version: 38171def3cd15e3b76eb156219b3d48704643899
- package: github.com/xeipuuv/gojsonpointer
version: e0fe6f68307607d540ed8eac07a342c33fa1b54a
- package: github.com/xeipuuv/gojsonreference
@ -37,4 +35,4 @@ import:
- package: github.com/xeipuuv/gojsonschema
version: d5336c75940ef31c9ceeb0ae64cf92944bccb4ee
- package: github.com/pkg/errors
version: ^0.7.0
version: 01fa4104b9c248c8945d14d9f128454d5b28d595

@ -15,7 +15,6 @@ else
| grep -v "github.com/olekukonko/ts" \
| grep -v "github.com/pkg/errors" \
| grep -v "github.com/stretchr/testify" \
| grep -v "github.com/technoweenie/go-contentaddressable" \
| grep -v "github.com/xeipuuv/gojsonreference" \
| grep -v "github.com/xeipuuv/gojsonschema" \
)

@ -1,20 +0,0 @@
Copyright (c) 2014 rick olson
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@ -1,49 +0,0 @@
# Content Addressable
Package contentaddressable contains tools for writing content addressable files.
Files are written to a temporary location, and only renamed to the final
location after the file's OID (Object ID) has been verified.
```go
filename := "path/to/01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
file, err := contentaddressable.NewFile(filename)
if err != nil {
panic(err)
}
defer file.Close()
file.Oid // 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
written, err := io.Copy(file, someReader)
if err == nil {
// Move file to final location if OID is verified.
err = file.Accept()
}
if err != nil {
panic(err)
}
```
See the [godocs](http://godoc.org/github.com/technoweenie/go-contentaddressable)
for details.
## Installation
$ go get github.com/technoweenie/go-contentaddressable
Then import it:
import "github.com/technoweenie/go-contentaddressable"
## Note on Patches/Pull Requests
1. Fork the project on GitHub.
2. Make your feature addition or bug fix.
3. Add tests for it. This is important so I don't break it in a future version
unintentionally.
4. Commit, do not mess with version or history. (if you want to have
your own version, that is fine but bump version in a commit by itself I can
ignore when I pull)
5. Send me a pull request. Bonus points for topic branches.

@ -1,28 +0,0 @@
/*
Package contentaddressable contains tools for writing content addressable files.
Files are written to a temporary location, and only renamed to the final
location after the file's OID (Object ID) has been verified.
filename := "path/to/01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"
file, err := contentaddressable.NewFile(filename)
if err != nil {
panic(err)
}
defer file.Close()
file.Oid // 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
written, err := io.Copy(file, someReader)
if err == nil {
// Move file to final location if OID is verified.
err = file.Accept()
}
if err != nil {
panic(err)
}
Currently SHA-256 is used for a file's OID.
*/
package contentaddressable

@ -1,131 +0,0 @@
package contentaddressable
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"hash"
"os"
"path/filepath"
)
var (
AlreadyClosed = errors.New("Already closed.")
HasData = errors.New("Destination file already has data.")
)
// File handles the atomic writing of a content addressable file. It writes to
// a temp file, and then renames to the final location after Accept().
type File struct {
Oid string
filename string
tempFilename string
file *os.File
tempFile *os.File
hasher hash.Hash
}
// NewFile initializes a content addressable file for writing. It opens both
// the given filename, and a temp filename in exclusive mode. The *File OID
// is taken from the base name of the given filename.
func NewFile(filename string) (*File, error) {
oid := filepath.Base(filename)
dir := filepath.Dir(filename)
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, err
}
tempFilename := filename + "-temp"
tempFile, err := os.OpenFile(tempFilename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
return nil, err
}
file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
cleanupFile(tempFile)
return nil, err
}
caw := &File{
Oid: oid,
filename: filename,
tempFilename: tempFilename,
file: file,
tempFile: tempFile,
hasher: sha256.New(),
}
return caw, nil
}
// Write sends data to the temporary file.
func (w *File) Write(p []byte) (int, error) {
if w.Closed() {
return 0, AlreadyClosed
}
w.hasher.Write(p)
return w.tempFile.Write(p)
}
// Accept verifies the written content SHA-256 signature matches the given OID.
// If it matches, the temp file is renamed to the original filename. If not,
// an error is returned.
func (w *File) Accept() error {
if w.Closed() {
return AlreadyClosed
}
sig := hex.EncodeToString(w.hasher.Sum(nil))
if sig != w.Oid {
return fmt.Errorf("Content mismatch. Expected OID %s, got %s", w.Oid, sig)
}
if err := cleanupFile(w.file); err != nil {
return err
}
w.file = nil
w.tempFile.Close()
err := os.Rename(w.tempFilename, w.filename)
w.Close()
return err
}
// Close cleans up the internal file objects.
func (w *File) Close() error {
if w.tempFile != nil {
if err := cleanupFile(w.tempFile); err != nil {
return err
}
w.tempFile = nil
}
if w.file != nil {
if err := cleanupFile(w.file); err != nil {
return err
}
w.file = nil
}
return nil
}
// Closed reports whether this file object has been closed.
func (w *File) Closed() bool {
if w.tempFile == nil || w.file == nil {
return true
}
return false
}
func cleanupFile(f *os.File) error {
err := f.Close()
if err := os.RemoveAll(f.Name()); err != nil {
return err
}
return err
}

@ -1,176 +0,0 @@
package contentaddressable
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"reflect"
"runtime"
)
var supOid = "a2b71d6ee8997eb87b25ab42d566c44f6a32871752c7c73eb5578cb1182f7be0"
func TestFile(t *testing.T) {
test := SetupFile(t)
defer test.Teardown()
filename := filepath.Join(test.Path, supOid)
aw, err := NewFile(filename)
assertEqual(t, nil, err)
n, err := aw.Write([]byte("SUP"))
assertEqual(t, nil, err)
assertEqual(t, 3, n)
by, err := ioutil.ReadFile(filename)
assertEqual(t, nil, err)
assertEqual(t, 0, len(by))
assertEqual(t, nil, aw.Accept())
by, err = ioutil.ReadFile(filename)
assertEqual(t, nil, err)
assertEqual(t, "SUP", string(by))
assertEqual(t, nil, aw.Close())
}
func TestFileMismatch(t *testing.T) {
test := SetupFile(t)
defer test.Teardown()
filename := filepath.Join(test.Path, "b2b71d6ee8997eb87b25ab42d566c44f6a32871752c7c73eb5578cb1182f7be0")
aw, err := NewFile(filename)
assertEqual(t, nil, err)
n, err := aw.Write([]byte("SUP"))
assertEqual(t, nil, err)
assertEqual(t, 3, n)
by, err := ioutil.ReadFile(filename)
assertEqual(t, nil, err)
assertEqual(t, 0, len(by))
err = aw.Accept()
if err == nil || !strings.Contains(err.Error(), "Content mismatch") {
t.Errorf("Expected mismatch error: %s", err)
}
by, err = ioutil.ReadFile(filename)
assertEqual(t, nil, err)
assertEqual(t, "", string(by))
assertEqual(t, nil, aw.Close())
_, err = ioutil.ReadFile(filename)
assertEqual(t, true, os.IsNotExist(err))
}
func TestFileCancel(t *testing.T) {
test := SetupFile(t)
defer test.Teardown()
filename := filepath.Join(test.Path, supOid)
aw, err := NewFile(filename)
assertEqual(t, nil, err)
n, err := aw.Write([]byte("SUP"))
assertEqual(t, nil, err)
assertEqual(t, 3, n)
assertEqual(t, nil, aw.Close())
for _, name := range []string{aw.filename, aw.tempFilename} {
if _, err := os.Stat(name); err == nil {
t.Errorf("%s exists?", name)
}
}
}
func TestFileLocks(t *testing.T) {
test := SetupFile(t)
defer test.Teardown()
filename := filepath.Join(test.Path, supOid)
aw, err := NewFile(filename)
assertEqual(t, nil, err)
assertEqual(t, filename, aw.filename)
assertEqual(t, filename+"-temp", aw.tempFilename)
files := []string{aw.filename, aw.tempFilename}
for _, name := range files {
if _, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0665); err == nil {
t.Errorf("Able to open %s!", name)
}
}
assertEqual(t, nil, aw.Close())
for _, name := range files {
f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0665)
assertEqualf(t, nil, err, "unable to open %s: %s", name, err)
cleanupFile(f)
}
}
func TestFileDuel(t *testing.T) {
test := SetupFile(t)
defer test.Teardown()
filename := filepath.Join(test.Path, supOid)
aw, err := NewFile(filename)
assertEqual(t, nil, err)
defer aw.Close()
if _, err := NewFile(filename); err == nil {
t.Errorf("Expected a file open conflict!")
}
}
func SetupFile(t *testing.T) *FileTest {
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Error getting wd: %s", err)
}
return &FileTest{filepath.Join(wd, "File"), t}
}
type FileTest struct {
Path string
*testing.T
}
func (t *FileTest) Teardown() {
if err := os.RemoveAll(t.Path); err != nil {
t.Fatalf("Error removing %s: %s", t.Path, err)
}
}
func assertEqual(t *testing.T, expected, actual interface{}) {
checkAssertion(t, expected, actual, "")
}
func assertEqualf(t *testing.T, expected, actual interface{}, format string, args ...interface{}) {
checkAssertion(t, expected, actual, format, args...)
}
func checkAssertion(t *testing.T, expected, actual interface{}, format string, args ...interface{}) {
if expected == nil {
if actual == nil {
return
}
} else if reflect.DeepEqual(expected, actual) {
return
}
_, file, line, _ := runtime.Caller(2) // assertEqual + checkAssertion
t.Logf("%s:%d\nExpected: %v\nActual: %v", file, line, expected, actual)
if len(args) > 0 {
t.Logf("! - "+format, args...)
}
t.FailNow()
}