Stefan Huber

last update

......@@ -4,21 +4,64 @@ Allows downloading of files referenced by a http/https URI inside a couchdb/pouc
# Usage
### In cordova projects
## Use it with pouchdb
### In electron projects
```typescript
import {ServiceLocator} from 'bsync-client/dist/browser-build';
Within the main process bsync needs to be integrated an initiated.
ServiceLocator.getConfig().setConfig('itemKey', 'type');
ServiceLocator.getConfig().setConfig('itemValue', 'asset');
import {Bsync} from 'bsync';
Bsync.init(ipcMain, filePath);
let localDb = new PouchDB('local-pouch-db');
let fileReplicator = ServiceLocator.getFileReplicator();
# Testing (Browser)
localDb.put({
_id : "_design/index_type",
views : {
type : {
map : function(doc) {
if (doc[fileReplicator.itemKey]) { emit(doc[fileReplicator.itemKey]); }
}.toString()
}
}
});
fileReplicator.once('complete', () => {
// All file are downloaded
});
db.query('index_type/type',{
include_docs : true,
key : fileReplicator.itemValue
}).then((res) => {
let docs = { docs : [] };
for (let r of res.rows) {
docs.docs.push(r.doc);
}
fileReplicator.pushChanges(docs);
fileReplicator.start();
});
```
## In electron projects
Within the `main process` bsync needs to be integrated an initiated.
```typescript
import {Bsync} from 'bsync';
Bsync.init(ipcMain, filePath);
```
# Testing (Browser/Node)
`npm test`
# Testing (Cordova)
`npm run test:cordova`
......
......@@ -8,10 +8,6 @@ export default {
format: 'umd',
globals: {
'rxjs' : 'Rx'
},
plugins: [
typescript() ,
globals(),
......
......@@ -9,13 +9,13 @@ export default {
entry: './src/browser-main.ts',
dest: './dist/browser-build.js',
format: 'cjs',
format: 'umd',
sourceMap: true ,
plugins: [
typescript(),
// globals(),
// builtins()
globals(),
builtins()
]
};
\ No newline at end of file
......
import typescript from 'rollup-plugin-typescript';
import builtins from 'rollup-plugin-node-builtins';
import globals from 'rollup-plugin-node-globals';
export default {
......@@ -10,6 +12,8 @@ export default {
format: 'cjs',
plugins: [
typescript()
typescript(),
globals(),
builtins()
]
};
\ No newline at end of file
......
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
'use strict';
var rxjs_Observable = require('rxjs/Observable');
var events = require('events');
var http = require('http');
var https = require('https');
var fs = require('fs');
var NodeFileHandler = (function () {
function __extends(d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
}
var Util = (function () {
function Util() {
}
Util.getNameHash = function (path) {
for (var r = 0, i = 0; i < path.length; i++) {
r = (r << 5) - r + path.charCodeAt(i), r &= r;
}
return "bsync_" + Math.abs(r);
};
/**
* index >= 0, localFile is in files
* index < 0, localFile is not in files
*/
Util.getFileIndex = function (files, localFile) {
for (var i = 0; i < files.length; i++) {
if (localFile == files[i].target) {
return i;
}
}
return -1;
};
Util.getFilesForCleanup = function (files, localFiles) {
var filesForCleanup = [];
for (var _i = 0, localFiles_1 = localFiles; _i < localFiles_1.length; _i++) {
var localFile = localFiles_1[_i];
var index = -1;
index = Util.getFileIndex(files, localFile);
if (index < 0) {
filesForCleanup.push(localFile);
}
else {
// splice for performance improvement only
files.splice(index, 1);
}
}
return filesForCleanup;
};
return Util;
}());
var NodeFileHandler = (function (_super) {
__extends(NodeFileHandler, _super);
function NodeFileHandler() {
_super.apply(this, arguments);
}
NodeFileHandler.prototype.selectProtocol = function (url) {
if (url.search(/^http:\/\//) === 0) {
......@@ -20,18 +68,19 @@ var NodeFileHandler = (function () {
}
};
NodeFileHandler.prototype.download = function (source, target) {
var _this = this;
var handler = this.selectProtocol(source);
return rxjs_Observable.Observable.create(function (subscriber) {
if (!handler) {
subscriber.error("No handler for source: " + source);
return;
this.emit("error", "No handler for source: " + source);
return this;
}
// file already exists and is not empty
if (fs.existsSync(target) && (fs.statSync(target)['size'] > 0)) {
subscriber.complete();
return;
this.emit("complete");
return this;
}
var file = fs.createWriteStream(target, { 'flags': 'a' });
else {
var file_1 = fs.createWriteStream(target, { 'flags': 'a' });
handler.get(source, function (response) {
var size = response.headers['content-length']; // in bytes
var prog = 0; // already downloaded
......@@ -39,33 +88,53 @@ var NodeFileHandler = (function () {
var nextProg = (1 / progCounts);
response.on('data', function (chunk) {
prog += chunk.length;
file.write(chunk, 'binary');
file_1.write(chunk, 'binary');
if ((prog / size) > nextProg) {
subscriber.next(prog / size);
_this.emit('progress', prog / size);
nextProg += (1 / progCounts);
}
});
response.on('end', function () {
file.end();
subscriber.complete();
response.once('end', function () {
file_1.end();
_this.emit('complete');
});
}).on('error', function (error) {
fs.unlink(target);
subscriber.error("Error while downloading: " + error);
_this.emit("error", "Error while downloading: " + error);
});
return this;
}
};
NodeFileHandler.prototype.cleanup = function (files, basePath) {
try {
var localFiles = fs.readdirSync(basePath);
Util.getFilesForCleanup(files, localFiles)
.forEach(function (file) {
fs.unlinkSync(basePath + "/" + file);
});
}
catch (e) {
}
return this;
};
return NodeFileHandler;
}());
}(events.EventEmitter));
var Bsync = (function () {
function Bsync() {
}
Bsync.configIpcMain = function (ipcMain, downloadDir) {
var nodeFileHander = new NodeFileHandler();
Bsync.configIpcMain = function (ipcMain, basePath) {
ipcMain.on('bsync-download', function (event, args) {
nodeFileHander.download(args.source, downloadDir + args.target)
.subscribe(function (progress) { event.sender.send('bsync-download-progress', progress); }, function (error) { event.sender.send('bsync-download-error', error); }, function () { event.sender.send('bsync-download-complete'); });
var nodeFileHander = new NodeFileHandler();
nodeFileHander.download(args.source, basePath + "/" + args.target)
.on('progress', function (progress) { event.sender.send('bsync-download-progress', progress); })
.once('error', function (error) { nodeFileHander.removeAllListeners(); event.sender.send('bsync-download-error', error); })
.once('complete', function () { nodeFileHander.removeAllListeners(); event.sender.send('bsync-download-complete'); });
});
ipcMain.on('bsync-cleanup', function (event, args) {
var nodeFileHandler = new NodeFileHandler();
nodeFileHandler.cleanup(args.files, basePath);
event.sender.send('bsync-cleanup-complete');
});
};
return Bsync;
......
This diff is collapsed. Click to expand it.
interface FileReplicator {
start(retries?:number) : void;
clear() : void;
cleanup() : void;
pushChanges(change: any) : void;
on(event:string, handler:(...params: any[]) => void) : FileReplicator;
once(event:string, handler:(...params: any[]) => void) : FileReplicator;
removeAllListeners() : FileReplicator;
}
declare var bsyncClient: {
CONFIG_TARGET_DIRECTORY : string ;
ServiceLocator : {
getConfig : () => {
getConfig : (key:string) => any;
setConfig : (key:string, value:any) => void;
hasConfig : (key:string) => boolean;
};
getFileReplicator : () => FileReplicator;
}
};
declare module "bsync-client" {
export = bsyncClient;
}
\ No newline at end of file
......@@ -3,6 +3,7 @@
"version": "1.0.0",
"description": "",
"main": "dist/browser-build.js",
"typings": "index.d.ts",
"scripts": {
"build": "npm run build:node && npm run build:browser",
"build:node": "rollup -c ./config/rollup.config.node.js",
......@@ -16,9 +17,6 @@
},
"author": "",
"license": "ISC",
"dependencies": {
"rxjs": "^5.0.2"
},
"devDependencies": {
"@types/jasmine": "^2.5.40",
"cordova": "^6.4.0",
......
#!/bin/bash
curl -X DELETE http://admin:admin@127.0.0.1:5984/pouch_test_db
rm -R ./.tmp
\ No newline at end of file
rm -rf ./.tmp
rm -rf ./.tmp-files
\ No newline at end of file
......
#!/bin/bash
rm -rf ./.tmp
rm -rf ./.tmp-files
curl -X DELETE http://admin:admin@127.0.0.1:5984/pouch_test_db
curl -X PUT http://admin:admin@127.0.0.1:5984/pouch_test_db
......
......@@ -7,15 +7,14 @@ echo "cordova $COMMAND $PLATFORM"
rollup --config ./config/rollup.config.cordova-test.js
cp ./spec/cordova-plugin-test/plugin.xml .tmp/plugin.xml
cp ./node_modules/rxjs/bundles/Rx.min.js .tmp/Rx.min.js
cd ..
rm -r ./bsync-client-test-app
rm -rf ./bsync-client-test-app
./bsync-client/node_modules/.bin/cordova create bsync-client-test-app
cd ./bsync-client-test-app
../bsync-client/node_modules/.bin/cordova platform add $PLATFORM
../bsync-client/node_modules/.bin/cordova plugin add ../bsync-client
# ../bsync-client/node_modules/.bin/cordova plugin add ../bsync-client
../bsync-client/node_modules/.bin/cordova plugin add ../bsync-client/.tmp
../bsync-client/node_modules/.bin/cordova plugin add cordova-plugin-test-framework
......@@ -27,4 +26,4 @@ else
cordova emulate $PLATFORM
fi
rm -R ../bsync-client/.tmp
\ No newline at end of file
rm -rf ../bsync-client/.tmp
\ No newline at end of file
......
......@@ -5,8 +5,6 @@ import {TestFileHandler} from './file-handler/test-file-handler';
ServiceLocator.addFileHandler(ENV_UNKNOWN, new TestFileHandler());
import '../src/browser-main';
declare var emit:any;
const dbUrl = 'http://admin:admin@localhost:5984/pouch_test_db';
......@@ -49,32 +47,71 @@ describe("Integration tests with couchdb", () => {
it("Should successfully download several files", (done) => {
TestFileHandler.setErrorRate(0);
ServiceLocator.getFileReplicator().init();
let replicator = ServiceLocator.getFileReplicator();
let index = 0;
localDb.replicate.from(dbUrl)
.on('file-replicator-complete', event => {
replicator.clear();
replicator
.on('file-complete', () => {
index++;
})
.on('complete', () => {
.once('complete', () => {
expect(index).toEqual(3);
done();
});
localDb.replicate.from(dbUrl)
.once('complete', () => {
localDb.query('index_type/type',{
include_docs : true,
key : replicator.itemValue
}).then((res) => {
let docs = [];
for (let r of res.rows) {
docs.push(r.doc);
}
replicator.pushChanges(docs);
replicator.start();
});
});
});
it("Should trigger errors, but successfully download with retries", (done) => {
TestFileHandler.setErrorRate(0.8);
ServiceLocator.getFileReplicator().init();
let replicator = ServiceLocator.getFileReplicator();
let errors = 0;
localDb.replicate.from(dbUrl)
.on('file-replicator-error', event => {
replicator.clear();
replicator
.on('file-error', () => {
errors++;
})
.on('complete', () => {
.once('complete', () => {
expect(errors).toBeGreaterThanOrEqual(1);
done();
});
localDb.replicate.from(dbUrl)
.once('complete', () => {
localDb.query('index_type/type',{
include_docs : true,
key : replicator.itemValue
}).then((res) => {
let docs = [];
for (let r of res.rows) {
docs.push(r.doc);
}
replicator.pushChanges(docs);
replicator.start();
});
});
});
});
......
......@@ -5,14 +5,11 @@
version="1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<name>bsync cordova test</name>
<name>bsync test in cordova environment</name>
<license>Apache 2.0 License</license>
<js-module src="Rx.min.js" name="Rx">
<clobbers target="Rx" />
</js-module>
<js-module src="cordova-test-build.js" name="tests">
</js-module>
<dependency id="cordova-plugin-file-transfer" url="https://github.com/apache/cordova-plugin-file-transfer" commit="master" />
</plugin>
\ No newline at end of file
......
import { CordovaDownloader } from '../src/file-handler/cordova-file-handler';
import { CordovaFileHandler } from '../src/file-handler/cordova-file-handler';
declare var cordova;
exports.defineAutoTests = function() {
let createFilesHelper = (filenames:Array<string>, successCallback, errorCallback) => {
let index = 0;
let create = () => {
if (index < filenames.length) {
console.log("try creating file: ", filenames[index]);
createFileHelper(filenames[index], create, errorCallback);
index += 1;
} else {
successCallback();
}
};
create();
};
let createFileHelper = (filename, successCallback, errorCallback) => {
window['requestFileSystem'](window['LocalFileSystem'].TEMPORARY, 0, (fs) => {
fs.root.getFile(filename, { create: true, exclusive: false },(fileEntry) => {
fileEntry.createWriter((fileWriter) => {
fileWriter.onwriteend = function() {
console.log("file created", filename);
successCallback();
};
fileWriter.onerror = function (e) {
console.error("file error", e);
errorCallback(e);
};
fileWriter.write(new Blob(['some arbitrary file data'], { type: 'text/plain' }));
});
}, errorCallback);
}, errorCallback);
};
jasmine.DEFAULT_TIMEOUT_INTERVAL = 100000;
describe("Cordova downloader", () => {
let downloader = new CordovaDownloader();
let handler = new CordovaFileHandler();
it("should download sample image from https source and store with new name", (done) => {
let source = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e1/FullMoon2010.jpg/800px-FullMoon2010.jpg";
let target = cordova.file.dataDirectory + "full-moon.jpg";
let source = "https://upload.wikimedia.org/wikipedia/commons/f/ff/Pizigani_1367_Chart_10MB.jpg";
// let target = cordova.file.dataDirectory + "full-moon.jpg";
let target = "cdvfile://localhost/temporary/full-moon.jpg";
let lastProgress = 0;
downloader.download(source, target)
.subscribe(
(progress: number) => {
handler.download(source, target)
.on('progress', (progress: number) => {
expect(progress).toBeGreaterThan(lastProgress);
lastProgress = progress;
} ,
(error:any) => {
}).once('error', (error:any) => {
fail();
} ,
() => {
}).once('complete', () => {
expect(lastProgress).toEqual(1);
window['resolveLocalFileSystemURL'](target, (entry:any) => {
......@@ -31,9 +64,31 @@ exports.defineAutoTests = function() {
expect(entry.name).toEqual("full-moon.jpg");
done();
});
}
);
});
});
it('should cleanup successfully', (done) => {
createFilesHelper(['file-1', 'file-2', 'file-3'], () => {
console.log("files are created, now start with cleanup");
handler.once('cleanup-complete', (files) => {
console.log(files);
expect(files.length).toBeGreaterThan(1);
done();
});
handler.once('cleanup-error', (files) => {
fail('cleanup error');
});
handler.cleanup([
{ source: '', target: 'file-1' } ,
{ source: '', target: 'file-2' }
], 'cdvfile://localhost/temporary/');
}, (error) => {
console.error(error);
fail('error while creating files');
});
});
});
......
import { Observable, Subscriber } from 'rxjs';
import { EventEmitter } from 'events';
import { FileHandler } from '../../src/api/file-handler';
export class TestFileHandler implements FileHandler {
export class TestFileHandler extends EventEmitter implements FileHandler {
protected static errorRate:number = 0;
......@@ -9,27 +9,30 @@ export class TestFileHandler implements FileHandler {
TestFileHandler.errorRate = rate;
}
download(source:string, target:string) : Observable<number> {
return Observable.create((subscriber:Subscriber<number>) => {
cleanup() { return this; }
download(source:string, target:string) {
let random = Math.random();
let error:boolean = random < TestFileHandler.errorRate;
let counter = 1;
if (error) {
subscriber.error("random error triggered");
return;
}
setTimeout(() => {
this.emit("error", "random error triggered");
},200);
} else {
let interval = setInterval(() => {
if (counter < 4) {
subscriber.next(counter * 25);
this.emit('progress', counter * 25);
} else {
subscriber.complete();
this.emit('complete');
clearInterval(interval);
}
++counter;
}, 10);
});
}
return this;
}
}
\ No newline at end of file
......
export * from './test/file-handler/node-file-handler';
export * from './test/file-replicator';
export * from './test/util';
\ No newline at end of file
......
......@@ -7,6 +7,10 @@ describe("Node Downloader", () => {
let nodeFileHandler = new NodeFileHandler();
beforeEach(() => {
nodeFileHandler.removeAllListeners();
});
it("should retrieve right handler", () => {
let httpHandler = nodeFileHandler.selectProtocol("http://someurl.com/image.jpg");
......@@ -23,27 +27,26 @@ describe("Node Downloader", () => {
it("should download sample image from https source and store with new name", (done) => {
let source = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e1/FullMoon2010.jpg/800px-FullMoon2010.jpg";
let target = ".tmp/full-moon.jpg";
let target = "./.tmp/full-moon-" + (new Date()).getTime() + ".jpg";
let lastProgress = 0;
nodeFileHandler.download(source, target)
.subscribe(
(progress: number) => {
nodeFileHandler
.on('progress', (progress: number) => {
expect(progress).toBeGreaterThan(lastProgress);
lastProgress = progress;
} ,
(error:any) => {} ,
() => {
})
.once('error', (error) => {})
.once('complete', () => {
expect(lastProgress).toEqual(1);
expect(fs.existsSync(target)).toBeTruthy();
done();
}
);
})
.download(source, target);
});
it('should not download if file with same name exists and bytesize > 0', (done) => {
let source = "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e1/FullMoon2010.jpg/800px-FullMoon2010.jpg";
let target = ".tmp/full-moon-sensless.jpg";
let target = "./.tmp/full-moon-sensless.jpg";
let lastProgress = 0;
let file = fs.createWriteStream(target, {'flags': 'a'});
......@@ -51,18 +54,32 @@ describe("Node Downloader", () => {
file.write(new Buffer(dummyData));
file.end(() => {
nodeFileHandler.download(source, target)
.subscribe(
() => { fail("progress should not be called"); } ,
() => {} ,
() => {
nodeFileHandler
.on('progress', () => { fail("progress should not be called"); })
.once('error', () => {})
.once('complete', () => {
expect(fs.existsSync(target)).toBeTruthy();
done();
}
);
})
.download(source, target);
});
});
it('should correctly cleanup files', () => {
let basePath = "./.tmp-files";
fs.mkdirSync(basePath);
fs.writeFileSync(basePath + "/file1", "some data for file 1");
fs.writeFileSync(basePath + "/file2", "some data for file 2");
fs.writeFileSync(basePath + "/file3", "some data for file 3");
fs.writeFileSync(basePath + "/file4", "some data for file 4");
nodeFileHandler.cleanup([
{ source : '' , target : 'file1' },
{ source : '' , target : 'file2' }], basePath);
let files = fs.readdirSync(basePath);
expect(files.length).toEqual(2);
});
});
\ No newline at end of file
......
import { FileReplicator } from '../../src/file-replicator';
describe("File Replicator", () => {
let fileReplicator = new FileReplicator();
let change = {
docs : [
let change = [
{ language: 'de', type : "asset", source : "http://someplace.com/icon.jpg" , target : "icon.jpg" } ,
{ language: 'de', type : "asset", source : "http://sampleuri.com/image.png" } ,
{ language: 'en', type : "asset", source : "https://secureasset.com/asset.mp3" , target : "music.mp3" }
]
};
];
beforeEach(() => {
fileReplicator.init();
fileReplicator.clear();
});
it("should contain several assets", () => {
......@@ -23,7 +20,7 @@ describe("File Replicator", () => {
});
it("should get correct asset names", () => {
let files = fileReplicator.prepareFiles(change.docs);
let files = fileReplicator.prepareFiles(change);
expect(files[0].target).toEqual("icon.jpg");
expect(files[1].target).toEqual("bsync_707608502");
......@@ -38,7 +35,7 @@ describe("File Replicator", () => {
return false;
};
let files = fileReplicator.prepareFiles(change.docs);
let files = fileReplicator.prepareFiles(change);
expect(files.length).toEqual(2);
expect(files[0].target).toEqual("icon.jpg");
......
import { Util } from '../../src/util';
describe('Util', () => {
let files = [
{ source : '' , target : 'file-1'} ,
{ source : '' , target : 'file-2'} ,
{ source : '' , target : 'file-3'} ,
];
it('should show files to remove', () => {
let cleanupFiles = Util.getFilesForCleanup(files, ['file-1','file-4']);
expect(cleanupFiles[0]).toEqual('file-4');
});
});
\ No newline at end of file
import { Observable } from 'rxjs/Observable';
import { File } from './file';
export interface FileHandler {
......@@ -8,6 +8,17 @@ export interface FileHandler {
* - if the download is in progress: trigger next (0-1 progress for percent of download)
* - if the download enters any error condition, trigger error and an already downloaded part of the file
*/
download(source:string, target:string) : Observable<number>;
download(source:string, target:string) : FileHandler;
/**
* Remove all files, which are not inside the files array from local storage
*/
cleanup(files: Array<File>, basePath?: string);
on(event:string, handler:(...params: any[]) => void) : FileHandler;
once(event:string, handler:(...params: any[]) => void) : FileHandler;
removeAllListeners() : FileHandler;
}
\ No newline at end of file
......
import { EventEmitter } from 'events';
import { ServiceLocator } from './service-locator';
export * from './service-locator';
export function loadBsyncPlugin (PouchDB) {
let pouchReplicate = PouchDB.replicate;
PouchDB.plugin((PouchDB) => {
PouchDB.replicate = function() {
let eventEmitter = new EventEmitter();
let emitter = pouchReplicate.apply(this, arguments);
let replicator = ServiceLocator.getFileReplicator();
let db = arguments[1];
replicator.once('final', event => {
eventEmitter.emit('complete');
eventEmitter.removeAllListeners();
});
replicator.on('error', event => {
eventEmitter.emit('file-replicator-error', event);
});
replicator.on('complete', event => {
eventEmitter.emit('file-replicator-complete', event);
});
replicator.on('progress', event => {
eventEmitter.emit('file-replicator-progress', event);
});
emitter.once('change', info => {
eventEmitter.emit('change', info);
});
emitter.once('complete', info => {
db.query('index_type/type',{
include_docs : true,
key : replicator.itemValue
}).then((res) => {
let docs = { docs : [] };
for (let r of res.rows) {
docs.docs.push(r.doc);
}
replicator.pushChanges(docs);
replicator.start();
}).catch(error => {
eventEmitter.emit('error', error);
});
});
emitter.once('error', (error) => {
eventEmitter.emit('error', error);
});
return eventEmitter;
};
});
};
if (typeof window !== 'undefined' && window['PouchDB']) {
loadBsyncPlugin(window['PouchDB']);
}
export * from './config';
......
......@@ -5,24 +5,26 @@ export const CONFIG_ITEM_TARGET_ATTRIBUTE = "itemTargetAttribute";
export const CONFIG_ITEM_VALIDATOR = "itemValidator";
export const CONFIG_RETRY_TIMEOUT = "retryTimeout";
export const CONFIG_FILE_HANDLER = "fileHandler";
export const CONFIG_TARGET_DIRECTORY = "targetDirectory";
export class Config {
protected config:any = {};
hasConfig(key:string) {
hasConfig(key:string) : boolean {
if (this.config[key]) {
return true;
}
return false;
}
getConfig(key:string) {
getConfig(key:string) : any {
return this.config[key];
}
setConfig(key:string, value:any) {
setConfig(key:string, value:any) : Config {
this.config[key] = value;
return this;
}
}
\ No newline at end of file
......
import { Util } from './../util';
import { EventEmitter } from 'events';
import { FileHandler } from '../api/file-handler';
import { File } from '../api/file';
declare var Rx;
export class CordovaDownloader implements FileHandler {
download(source:string, target:string) : Rx.Observable<number> {
return Rx.Observable.create((subscriber:Rx.Subscriber<number>) => {
if (!window['FileTransfer']) {
subscriber.error("Cordova FileTransfer object undefined");
}
export class CordovaFileHandler extends EventEmitter implements FileHandler {
triggerFileTransfer(source:string, target:string) {
let fileTransfer = new window['FileTransfer']();
fileTransfer.onprogress = (progress:ProgressEvent) => {
subscriber.next(progress.loaded / progress.total);
this.emit('progress', progress.loaded / progress.total);
};
fileTransfer.download(
source ,
target ,
(entry:any) => {
subscriber.complete();
this.emit('complete', entry);
} ,
(error:any) => {
subscriber.error(error);
this.emit('error', error);
},
true
);
}
download(source:string, target:string) {
if (!window['FileTransfer']) {
this.emit('error','Cordova FileTransfer object undefined');
}
window['resolveLocalFileSystemURL'](target, (entry:any) => {
entry.getMetadata((metadata) => {
if (metadata.size > 0) {
this.emit('complete', entry);
} else {
// file empty trigger transfer
this.triggerFileTransfer(source,target);
}
}, () => {
// cannot read metadata trigger transfer
this.triggerFileTransfer(source,target);
});
}, () => {
// file not found, so download it
this.triggerFileTransfer(source,target);
});
return this;
}
cleanup(files:Array<File>, basePath:string) : Promise<void> {
let filesForCleanup = [];
return new Promise<void>((resolve, reject) => {
if (window['resolveLocalFileSystemURL']) {
window['resolveLocalFileSystemURL'](basePath, (entry) => {
let reader = entry.createReader();
reader.readEntries((entries) => {
for (let e of entries) {
if (e && e.isFile) {
if (Util.getFileIndex(files, e.name) < 0) {
filesForCleanup.push(e);
}
}
}
let index = 0;
let error = false;
let cleanupError = (error) => {
this.emit('cleanup-error', error);
reject();
error = true;
};
let cleanupNext = () => {
if (index < filesForCleanup.length && !error) {
filesForCleanup[index].remove(cleanupNext, cleanupError);
index += 1;
} else if (!error) {
this.emit('cleanup-complete', filesForCleanup);
resolve();
}
};
cleanupNext();
}, (error) => { this.emit('cleanup-error', error); reject(); });
});
}
});
}
......
import { Observable, Subscriber } from 'rxjs';
import { EventEmitter } from 'events';
import { FileHandler } from '../api/file-handler';
import { File } from '../api/file';
export class ElectronFileHandler implements FileHandler {
export class ElectronFileHandler extends EventEmitter implements FileHandler {
constructor (private ipcRenderer:any) {
super();
}
download(source:string, target:string) : Observable<number> {
return Observable.create((subscriber:Subscriber<number>) => {
download(source:string, target:string) {
this.ipcRenderer.once('bsync-download-complete', () => {
this.ipcRenderer.removeAllListeners('bsync-download-progress');
this.ipcRenderer.removeAllListeners('bsync-download-error');
subscriber.complete();
this.emit('complete');
});
this.ipcRenderer.on('bsync-download-progress', (progress:number) => {
subscriber.next(progress);
this.emit('progress', progress);
});
this.ipcRenderer.once('bsync-download-error', (error:any) => {
this.ipcRenderer.removeAllListeners('bsync-download-progress');
this.ipcRenderer.removeAllListeners('bsync-download-complete');
subscriber.error(error);
this.emit('error', error);
});
this.ipcRenderer.send('bsync-download', {
source : source ,
target : target
});
return this;
}
cleanup(files:Array<File>) : Promise<void> {
return new Promise<void>((resolve, reject) => {
this.ipcRenderer.once('bsync-cleanup-complete', () => {
this.emit('cleanup-complete');
resolve();
});
this.ipcRenderer.send('bsync-cleanup', files);
});
}
......
import { Observable } from 'rxjs/Observable';
import { Subscriber } from 'rxjs/Subscriber';
import { Util } from './../util';
import { EventEmitter } from 'events';
import { FileHandler } from '../api/file-handler';
import { File } from '../api/file';
import * as http from 'http';
import * as https from 'https';
import * as fs from 'fs';
export class NodeFileHandler implements FileHandler {
export class NodeFileHandler extends EventEmitter implements FileHandler {
selectProtocol(url:string) : any {
if (url.search(/^http:\/\//) === 0) {
......@@ -17,23 +18,20 @@ export class NodeFileHandler implements FileHandler {
}
}
download(source:string, target:string) : Observable<number> {
download(source:string, target:string) {
let handler = this.selectProtocol(source);
return Observable.create((subscriber:Subscriber<number>) => {
if (!handler) {
subscriber.error("No handler for source: " + source);
return;
this.emit("error","No handler for source: " + source);
return this;
}
// file already exists and is not empty
if (fs.existsSync(target) && (fs.statSync(target)['size'] > 0)) {
subscriber.complete();
return;
}
this.emit("complete");
return this;
} else {
let file = fs.createWriteStream(target, {'flags': 'a'});
handler.get(source, (response) => {
......@@ -47,23 +45,36 @@ export class NodeFileHandler implements FileHandler {
file.write(chunk, 'binary');
if ((prog / size) > nextProg) {
subscriber.next(prog / size);
this.emit('progress',prog / size);
nextProg += (1 / progCounts);
}
});
response.on('end', () => {
response.once('end', () => {
file.end();
subscriber.complete();
this.emit('complete');
});
}).on('error', (error) => {
fs.unlink(target);
subscriber.error("Error while downloading: " + error);
this.emit("error", "Error while downloading: " + error);
});
return this;
}
}
cleanup(files:Array<File>, basePath?:string) {
try {
let localFiles = fs.readdirSync(basePath);
Util.getFilesForCleanup(files, localFiles)
.forEach((file) => {
fs.unlinkSync(basePath + "/" + file);
});
} catch (e) {
}
return this;
}
}
\ No newline at end of file
......
import {FileHandler} from './api/file-handler';
import {File} from './api/file';
import {Util} from './util';
import {EventEmitter} from 'events';
import { FileHandler } from './api/file-handler';
import { File } from './api/file';
import { Util } from './util';
import { EventEmitter } from 'events';
export class FileReplicator extends EventEmitter {
......@@ -9,46 +9,52 @@ export class FileReplicator extends EventEmitter {
super();
}
protected _files:Array<File> = [];
protected _files: Array<File> = [];
protected _itemValidator: (item:any) => boolean = null;
protected _fileHandler:FileHandler = null;
protected _retryTimeout:number = 0;
protected _itemValidator: (item: any) => boolean = null;
protected _fileHandler: FileHandler = null;
protected _retryTimeout: number = 100;
protected _retries: number = 10;
protected _itemKey = "type";
protected _itemValue = "asset";
protected _itemSourceAttribute = "source";
protected _itemTargetAttribute = "target";
protected _targetDirectory = "";
get files(): Array<File> {
return this._files;
}
set fileHandler (handler:FileHandler) {
set fileHandler(handler: FileHandler) {
this._fileHandler = handler;
}
set retryTimeout (timeout:number) {
get fileHandler() : FileHandler {
return this._fileHandler;
}
set retryTimeout(timeout: number) {
this._retryTimeout = timeout;
}
set itemValidator(validator:(item:any) => boolean) {
set itemValidator(validator: (item: any) => boolean) {
this._itemValidator = validator;
}
set itemKey(key:string) {
set itemKey(key: string) {
this._itemKey = key;
}
set itemValue(value:string) {
set itemValue(value: string) {
this._itemValue = value;
}
set itemSourceAttribute(sourceAttribute:string) {
set itemSourceAttribute(sourceAttribute: string) {
this._itemSourceAttribute = sourceAttribute;
}
set itemTargetAttribute(targetAttribute:string) {
set itemTargetAttribute(targetAttribute: string) {
this._itemTargetAttribute = targetAttribute;
}
......@@ -68,18 +74,26 @@ export class FileReplicator extends EventEmitter {
return this._itemTargetAttribute;
}
init(files: Array<File> = []) {
set targetDirectory(targetDirectory: string) {
this._targetDirectory = targetDirectory;
}
get targetDirectory() {
return this._targetDirectory;
}
clear(files: Array<File> = []) {
this._files = files;
}
/**
* change from pouchdb replicate
*/
pushChanges(change:any) {
let items:Array<any> = [];
pushChanges(docs: Array<any>) {
let items: Array<any> = [];
if (change && change.docs && change.docs.length > 0) {
for (let item of change.docs) {
if (docs && docs.length > 0) {
for (let item of docs) {
if (item[this._itemKey] && item[this._itemKey] === this._itemValue) {
items.push(item);
}
......@@ -93,35 +107,35 @@ export class FileReplicator extends EventEmitter {
}
}
downloadFiles(files:Array<File>, fileHandler:FileHandler, index:number = 0) {
downloadFiles(files: Array<File>, fileHandler: FileHandler, index: number = 0) {
if (index >= files.length) {
return;
}
this.emit('start', { progress: 0, index : index, length : files.length });
this.emit('start', { progress: 0, index: index, length: files.length });
fileHandler
.download(files[index].source, files[index].target)
.subscribe(
progress => {
this.emit('progress', { progress : progress, index : index, length : files.length })
} ,
error => {
this.emit('error', { progress : 0, index : index, length : files.length, error: error });
} ,
() => {
this.emit('complete', { progress : 100 , index : index, length : files.length });
this.downloadFiles(files, fileHandler, index+1);
}
);
}
prepareFiles(items: Array<any>) : Array<File> {
.on('progress', (progress) => {
this.emit('file-progress', { progress: progress, index: index, length: files.length })
})
.once('error', error => {
this.emit('file-error', { progress: 0, index: index, length: files.length, error: error });
fileHandler.removeAllListeners();
})
.once('complete', () => {
this.emit('file-complete', { progress: 100, index: index, length: files.length });
fileHandler.removeAllListeners();
this.downloadFiles(files, fileHandler, index + 1);
})
.download(files[index].source, this.targetDirectory + files[index].target);
}
prepareFiles(items: Array<any>): Array<File> {
let output = [];
for (let item of items) {
if (item[this._itemSourceAttribute] && (!this._itemValidator || this._itemValidator(item))) {
let file = { source : item[this._itemSourceAttribute] , target : '' };
let file = { source: item[this._itemSourceAttribute], target: '' };
if (item[this._itemTargetAttribute]) {
file.target = item[this._itemTargetAttribute];
......@@ -136,29 +150,47 @@ export class FileReplicator extends EventEmitter {
return output;
}
start() {
this.on('complete', (event:any) => {
start(retries:number = 10) {
this._retries = retries;
this.on('file-complete', (event: any) => {
if ((event.index + 1) >= event.length) {
this.replicationFinalized(event.index);
}
});
this.on('error', (event:any) => {
this.on('file-error', (event: any) => {
this.replicationFinalized(event.index);
});
if (this._fileHandler && this._files.length > 0) {
this.downloadFiles(this._files, this._fileHandler);
} else {
this.emit('complete');
}
}
cleanup() {
this.fileHandler
.cleanup(this._files, this._targetDirectory)
.then(() => {
this.emit('cleanup-complete');
}).catch(() => {
this.emit('cleanup-error');
});
}
replicationFinalized(lastIndex:number) {
if (lastIndex+1 >= this._files.length) { // all finished
this._files = [];
this.emit('final');
} else if (this._retryTimeout > 0) { // restart after last success
this._files.splice(0,lastIndex);
replicationFinalized(lastIndex: number) {
if (lastIndex + 1 >= this._files.length) { // all finished
this.emit('complete');
} else if (this._retries > 0) { // restart after last success
this._files.splice(0, lastIndex);
setTimeout(() => {
this._retries =- 1;
this.downloadFiles(this._files, this._fileHandler);
}, this._retryTimeout);
} else {
this.emit('error');
}
}
......
......@@ -2,17 +2,22 @@ import { NodeFileHandler } from './file-handler/node-file-handler';
export default class Bsync {
static configIpcMain(ipcMain: any, downloadDir:string) {
let nodeFileHander = new NodeFileHandler();
static configIpcMain(ipcMain: any, basePath: string) {
ipcMain.on('bsync-download', (event, args) => {
nodeFileHander.download(args.source, downloadDir + args.target)
.subscribe(
(progress:number) => { event.sender.send('bsync-download-progress', progress); } ,
(error:any) => { event.sender.send('bsync-download-error', error); } ,
() => { event.sender.send('bsync-download-complete'); }
);
let nodeFileHander = new NodeFileHandler();
nodeFileHander.download(args.source, basePath + "/" + args.target)
.on('progress', (progress:number) => { event.sender.send('bsync-download-progress', progress); })
.once('error', (error) => { nodeFileHander.removeAllListeners(); event.sender.send('bsync-download-error', error); })
.once('complete', () => { nodeFileHander.removeAllListeners(); event.sender.send('bsync-download-complete'); });
});
ipcMain.on('bsync-cleanup', (event, args) => {
let nodeFileHandler = new NodeFileHandler();
nodeFileHandler.cleanup(args.files, basePath);
event.sender.send('bsync-cleanup-complete');
});
}
}
\ No newline at end of file
......
import {FileHandler} from './api/file-handler';
import {ElectronFileHandler} from './file-handler/electron-file-handler';
import {CordovaDownloader} from './file-handler/cordova-file-handler';
import {CordovaFileHandler} from './file-handler/cordova-file-handler';
import {FileReplicator} from './file-replicator';
import {
Config,
......@@ -8,7 +8,8 @@ import {
CONFIG_ITEM_KEY,
CONFIG_ITEM_VALUE,
CONFIG_ITEM_TARGET_ATTRIBUTE,
CONFIG_ITEM_SOURCE_ATTRIBUTE
CONFIG_ITEM_SOURCE_ATTRIBUTE,
CONFIG_TARGET_DIRECTORY
} from './config';
export const ENV_ELECTRON = "electron";
......@@ -56,7 +57,7 @@ export class ServiceLocator {
}
if (environment === ENV_CORDOVA) {
return new CordovaDownloader();
return new CordovaFileHandler();
}
return null;
......@@ -64,10 +65,9 @@ export class ServiceLocator {
static getFileReplicator() : FileReplicator {
if (!ServiceLocator.fileReplicator) {
ServiceLocator.fileReplicator = new FileReplicator();
ServiceLocator.fileReplicator.fileHandler = ServiceLocator.getFileHandler();
}
if (ServiceLocator.getConfig().hasConfig(CONFIG_RETRY_TIMEOUT)) {
ServiceLocator.fileReplicator.retryTimeout = ServiceLocator.getConfig().getConfig(CONFIG_RETRY_TIMEOUT);
......@@ -84,6 +84,8 @@ export class ServiceLocator {
if (ServiceLocator.getConfig().hasConfig(CONFIG_ITEM_TARGET_ATTRIBUTE)) {
ServiceLocator.fileReplicator.itemTargetAttribute = ServiceLocator.getConfig().getConfig(CONFIG_ITEM_TARGET_ATTRIBUTE);
}
if (ServiceLocator.getConfig().hasConfig(CONFIG_TARGET_DIRECTORY)) {
ServiceLocator.fileReplicator.targetDirectory = ServiceLocator.getConfig().getConfig(CONFIG_TARGET_DIRECTORY);
}
return ServiceLocator.fileReplicator;
......
import { File } from './api/file';
export class Util {
static getNameHash(path:string) {
......@@ -7,4 +9,36 @@ export class Util {
return "bsync_" + Math.abs(r);
}
/**
* index >= 0, localFile is in files
* index < 0, localFile is not in files
*/
static getFileIndex(files:Array<File>, localFile:string) : number {
for (let i = 0; i < files.length; i++) {
if (localFile == files[i].target) {
return i;
}
}
return -1;
}
static getFilesForCleanup(files:Array<File>, localFiles:Array<string>) : Array<string> {
let filesForCleanup = [];
for (let localFile of localFiles) {
let index = -1;
index = Util.getFileIndex(files, localFile);
if (index < 0) {
filesForCleanup.push(localFile);
} else {
// splice for performance improvement only
files.splice(index,1);
}
}
return filesForCleanup;
}
}
\ No newline at end of file
......