Cloud sync rework

* reworked the hole cloud sync controller
this will make it more save to use the cloud sync
THIS IS WORK IN PROGRESS, NOT READY FOR DAILY USE!
This commit is contained in:
Jannik 2017-12-12 23:52:54 +01:00
parent 9b443dfeae
commit 42b8434a58
5 changed files with 201 additions and 271 deletions

View File

@ -141,6 +141,7 @@ public class Main extends Application {
mainWindowController.setColor("00a8cc");
mainWindowController.setAutoUpdate(false);
mainWindowController.setxPosHelper(0);
mainWindowController.setLastLocalSync(0);
mainWindowController.saveSettings();
Runtime.getRuntime().exec("java -jar cemu_UI.jar"); //start again (preventing Bugs)
System.exit(0); //finishes itself
@ -174,10 +175,11 @@ public class Main extends Application {
// loading settings and initialize UI, dbController.main() loads all databases
mainWindowController.init();
mainWindowController.dbController.init();
// if cloud sync is activated start sync
if(mainWindowController.isCloudSync()) {
cloudController.initializeConnection(mainWindowController.getCloudService(), mainWindowController.getCemuPath());
cloudController.stratupCheck(mainWindowController.getCloudService(), mainWindowController.getCemuPath());
cloudController.sync(mainWindowController.getCloudService(), mainWindowController.getCemuPath(), directory.getPath());
}
mainWindowController.addUIData();
@ -340,4 +342,12 @@ public class Main extends Application {
public void setPane(AnchorPane pane) {
this.pane = pane;
}
public File getDirectory() {
return directory;
}
public void setDirectory(File directory) {
this.directory = directory;
}
}

View File

@ -284,6 +284,7 @@ public class MainWindowController {
private int oldXPosHelper;
private int selectedUIDataIndex;
private int selected;
private long lastLocalSync;
private double windowWidth;
private double windowHeight;
private DirectoryChooser directoryChooser = new DirectoryChooser();
@ -1034,8 +1035,9 @@ public class MainWindowController {
cloudSync = false;
} else {
String headingText = "activate cloud savegame sync (beta)";
String bodyText = "You just activate the cloud savegame sync function of cemu_UI, "
+ "\nwhich is currently in beta. Are you sure you want to do this?";
String bodyText = "WARNING this is a completly WIP cloud save integration, "
+ "\nit's NOT recomended to use this!!\n"
+ "\nUse it on your own risk and backup everthing before!";
EventHandler<ActionEvent> okayAction = new EventHandler<ActionEvent>(){
@Override
@ -1050,7 +1052,7 @@ public class MainWindowController {
public void run() {
if (main.getCloudController().initializeConnection(getCloudService(), getCemuPath())) {
main.getCloudController().sync(getCloudService(), getCemuPath());
main.getCloudController().sync(getCloudService(), getCemuPath(), main.getDirectory().getPath());
saveSettings();
} else {
cloudSyncToggleBtn.setSelected(false);
@ -1553,7 +1555,7 @@ public class MainWindowController {
} else {
props.setProperty("cloudService", getCloudService());
}
props.setProperty("folderID", main.getCloudController().getFolderID(getCloudService()));
props.setProperty("lastLocalSync", String.valueOf(getLastLocalSync()));
props.setProperty("windowWidth", String.valueOf(mainAnchorPane.getWidth()));
props.setProperty("windowHeight", String.valueOf(mainAnchorPane.getHeight()));
if(System.getProperty("os.name").equals("Linux")){
@ -1641,10 +1643,10 @@ public class MainWindowController {
}
try {
main.getCloudController().setFolderID(props.getProperty("folderID"), getCloudService());
setLastLocalSync(Long.parseLong(props.getProperty("lastLocalSync")));
} catch (Exception e) {
LOGGER.error("could not load folderID, disable cloud sync. Please contact an developer", e);
setCloudSync(false);
LOGGER.error("could not load lastSuccessSync, setting default instead", e);
setLastLocalSync(0);
}
try {
@ -1828,6 +1830,14 @@ public class MainWindowController {
this.xPosHelper = xPosHelper;
}
public long getLastLocalSync() {
return lastLocalSync;
}
public void setLastLocalSync(long lastLocalSync) {
this.lastLocalSync = lastLocalSync;
}
public boolean isFullscreen() {
return fullscreen;
}

View File

@ -22,6 +22,7 @@
package com.cemu_UI.application;
import java.io.IOException;
import java.time.Instant;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -91,9 +92,11 @@ public class playGame extends Thread{
// System.out.println(mainWindowController.getCemuPath()+"/mlc01/emulatorSave/"+);
//sync savegame with cloud service
if(mainWindowController.isCloudSync()) {
mainWindowController.main.getCloudController().sync(mainWindowController.getCloudService(), mainWindowController.getCemuPath());
}
if (mainWindowController.isCloudSync()) {
mainWindowController.setLastLocalSync(Instant.now().getEpochSecond());
mainWindowController.main.getCloudController().sync(mainWindowController.getCloudService(),
mainWindowController.getCemuPath(), mainWindowController.main.getDirectory().getPath());
}
}catch (IOException | InterruptedException e){
e.printStackTrace();

View File

@ -22,7 +22,12 @@
package com.cemu_UI.controller;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.time.Instant;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -31,6 +36,8 @@ import com.cemu_UI.application.MainWindowController;
import com.cemu_UI.vendorCloudController.GoogleDriveController;
import javafx.application.Platform;
import net.lingala.zip4j.core.ZipFile;
import net.lingala.zip4j.exception.ZipException;
public class CloudController {
@ -63,118 +70,136 @@ public class CloudController {
LOGGER.info("cloud initialisation done!");
return success;
}
public void stratupCheck(String cloudService, String cemuDirectory) {
if(cloudService.equals("GoogleDrive")) {
LOGGER.info("starting startup check google drive ...");
try {
if (!googleDriveController.checkFolder()) {
googleDriveController.creatFolder();
mwc.saveSettings();
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
Platform.runLater(() -> {
mwc.getPlayBtn().setText("syncing...");
});
googleDriveController.uploadAllFiles();
Platform.runLater(() -> {
mwc.getPlayBtn().setText("play");
});
}
});
thread.start();
} else {
sync(cloudService, cemuDirectory);
}
} catch (IOException e) {
LOGGER.error("google drive startup check failed", e);
}
}
if(cloudService.equals("Dropbox")) {
}
}
public void sync(String cloudService, String cemuDirectory) {
//running sync in a new thread, instead of blocking the main thread
/**
* to trigger a new sync set the mwc LastLocalSync to the actual time and call the sync method
* @param cloudService
* @param cemuDirectory
* @param cemu_UIDirectory
*/
public void sync(String cloudService, String cemuDirectory, String cemu_UIDirectory) {
// running sync in a new thread, instead of blocking the main thread
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
Platform.runLater(() -> {
mwc.getPlayBtn().setText("syncing...");
});
LOGGER.info("starting synchronization in new thread ...");
if(cloudService.equals("GoogleDrive")) {
try {
googleDriveController.sync(cemuDirectory);
} catch (IOException e) {
LOGGER.error("google drive synchronization failed", e);
}
}
if(cloudService.equals("Dropbox")) {
}
Platform.runLater(() -> {
mwc.getPlayBtn().setText("play");
mwc.saveSettings();
});
LOGGER.info("synchronization successful!");
}
});
try {
Platform.runLater(() -> {
mwc.getPlayBtn().setDisable(true);
mwc.getPlayBtn().setText("syncing...");
});
LOGGER.info("starting synchronization in new thread ...");
// zip the saves folder
File zipFile = zipSavegames(cemu_UIDirectory, cemuDirectory);
// upload the zip
switch (cloudService) {
// use GoogleDriveController
case "GoogleDrive":
LOGGER.info("using GoogleDriveController");
long lastCloudSync = googleDriveController.getLastCloudSync();
if (!googleDriveController.checkFolder()) {
LOGGER.info("cloud sync folder dosen't exist, creating one!");
googleDriveController.creatFolder();
googleDriveController.uploadZipFile(zipFile);
} else if (mwc.getLastLocalSync() > lastCloudSync) {
LOGGER.info("local is new, going to upload zip");
googleDriveController.uploadZipFile(zipFile);
} else if(mwc.getLastLocalSync() < lastCloudSync) {
LOGGER.info("cloud is new, going to download zip");
unzipSavegames(cemuDirectory, googleDriveController.downloadZipFile(cemu_UIDirectory));
mwc.setLastLocalSync(lastCloudSync);
break;
} else {
LOGGER.info("nothing to do");
break;
}
mwc.setLastLocalSync(Long.parseLong(zipFile.getName().substring(0, zipFile.getName().length()-4))); // set time of last sucessfull sync
break;
case "Dropbox":
break;
default:
LOGGER.warn("no cloud vendor found!");
break;
}
zipFile.delete(); // delete zipfile in cem_UI directory
Platform.runLater(() -> {
mwc.getPlayBtn().setText("play");
mwc.getPlayBtn().setDisable(false);
mwc.saveSettings();
});
LOGGER.info("synchronization successful!");
} catch (Exception e) {
// TODO: handle exception
}
}
});
thread.start();
}
void uploadFile(String cloudService, File file) {
//running uploadFile in a new thread, instead of blocking the main thread
new Thread() {
@Override
public void run() {
LOGGER.info("starting uploadFile in new thread ...");
if(cloudService.equals("GoogleDrive")) {
try {
googleDriveController.uploadFile(file);
} catch (IOException e) {
LOGGER.error("google drive uploadFile failed" ,e);
}
}
if(cloudService.equals("Dropbox")) {
}
}
}.start();
}
public String getFolderID(String cloudService) {
String folderID = "";
if (cloudService != null) {
if(cloudService.equals("GoogleDrive")) {
folderID = googleDriveController.getFolderID();
}
if(cloudService.equals("Dropbox")) {
}
}
return folderID;
private File zipSavegames(String cemu_UIDirectory, String cemuDirectory) throws Exception {
long unixTimestamp = Instant.now().getEpochSecond();
FileOutputStream fos = new FileOutputStream(cemu_UIDirectory + "/" + unixTimestamp + ".zip");
ZipOutputStream zos = new ZipOutputStream(fos);
addDirToZipArchive(zos, new File(cemuDirectory + "/mlc01/usr/save"), null);
zos.flush();
fos.flush();
zos.close();
fos.close();
return new File(cemu_UIDirectory + "/" + unixTimestamp + ".zip");
}
private static void addDirToZipArchive(ZipOutputStream zos, File fileToZip, String parrentDirectoryName) throws Exception {
if (fileToZip == null || !fileToZip.exists()) {
return;
}
public void setFolderID(String folderID, String cloudService) {
if (cloudService != null) {
if (cloudService.equals("GoogleDrive")) {
googleDriveController.setFolderID(folderID);
}
if (cloudService.equals("Dropbox")) {
}
String zipEntryName = fileToZip.getName();
if (parrentDirectoryName!=null && !parrentDirectoryName.isEmpty()) {
zipEntryName = parrentDirectoryName + "/" + fileToZip.getName();
}
if (fileToZip.isDirectory()) {
// System.out.println("+" + zipEntryName);
for (File file : fileToZip.listFiles()) {
addDirToZipArchive(zos, file, zipEntryName);
}
} else {
// System.out.println(" " + zipEntryName);
byte[] buffer = new byte[1024];
FileInputStream fis = new FileInputStream(fileToZip);
zos.putNextEntry(new ZipEntry(zipEntryName));
int length;
while ((length = fis.read(buffer)) > 0) {
zos.write(buffer, 0, length);
}
zos.closeEntry();
fis.close();
}
}
private void unzipSavegames(String cemuDirectory, File outputFile) {
try {
ZipFile zipFile = new ZipFile(outputFile);
zipFile.extractAll(cemuDirectory + "/mlc01/usr");
outputFile.delete();
LOGGER.info("unzip successfull");
} catch (ZipException e) {
LOGGER.error("an error occurred during unziping the file!", e);
}
}
}

View File

@ -21,17 +21,12 @@
package com.cemu_UI.vendorCloudController;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -56,15 +51,11 @@ import com.google.api.services.drive.model.FileList;
public class GoogleDriveController {
Drive service;
private String saveDirectory;
private String folderID;
private ArrayList<java.io.File> localSavegames = new ArrayList<>();
private ArrayList<File> cloudSavegames = new ArrayList<>();
private ArrayList<String> localSavegamesName = new ArrayList<>();
private ArrayList<String> cloudSavegamesName = new ArrayList<>();
private File downloadFile;
private static final Logger LOGGER = LogManager.getLogger(GoogleDriveController.class.getName());
private final String APPLICATION_NAME ="cemu_Ui Drive API Controller"; //TODO add Google
private final String APPLICATION_NAME ="cemu_Ui Google Drive API Controller";
//Directory to store user credentials for this application
private final java.io.File DATA_STORE_DIR = new java.io.File(System.getProperty("user.home"), ".credentials/cemu_UI_credential");
@ -127,67 +118,10 @@ public class GoogleDriveController {
public void main(String cemuDirectory) throws IOException {
java.io.File dir = new java.io.File(cemuDirectory + "/mlc01/usr/save");
service = getDriveService();
// cemu >= 1.11 uses /mlc01/usr/save/... instead of /mlc01/emulatorSave/...
if (dir.exists()) {
LOGGER.info("using new save path");
saveDirectory = cemuDirectory + "/mlc01/usr/save";
} else {
LOGGER.info("using old save path");
saveDirectory = cemuDirectory + "/mlc01/emulatorSave";
}
service = getDriveService();
}
public void sync(String cemuDirectory) throws IOException {
//in case there is no folderID saved, look it up first
if (getFolderID() == "" || getFolderID() == null) {
getSavegamesFolderID();
}
getLocalSavegames();
getCloudSavegames();
// download files from cloud which don't exist locally
for (int i = 0; i < cloudSavegames.size(); i++) {
// if the file exists locally, check which one is newer
if (localSavegamesName.contains(cloudSavegames.get(i).getName())) {
int localSavegamesNumber = localSavegamesName.indexOf(cloudSavegames.get(i).getName());
long localModified = new DateTime(localSavegames.get(localSavegamesNumber).lastModified()).getValue();
long cloudModified = cloudSavegames.get(i).getModifiedTime().getValue();
FileInputStream fis = new FileInputStream(localSavegames.get(localSavegamesNumber));
if (cloudSavegames.get(i).getMd5Checksum().equals(org.apache.commons.codec.digest.DigestUtils.md5Hex(fis))) {
LOGGER.info("both files are the same, nothing to do");
} else {
if (localModified >= cloudModified) {
LOGGER.info("local is newer");
updateFile(cloudSavegames.get(i), localSavegames.get(localSavegamesNumber));
} else {
LOGGER.info("cloud is newer");
downloadFile(cloudSavegames.get(i));
}
}
} else {
LOGGER.info("file doesn't exist locally");
downloadFile(cloudSavegames.get(i));
}
}
// upload file to cloud which don't exist in the cloud
for (int j = 0; j < localSavegames.size(); j++) {
if (!cloudSavegamesName.contains(localSavegamesName.get(j))) {
LOGGER.info("file doesn't exist in the cloud");
uploadFile(localSavegames.get(j));
}
}
}
//create a folder in google drive
// create a folder in google drive
public void creatFolder() throws IOException {
LOGGER.info("creating new folder");
File fileMetadata = new File();
@ -199,8 +133,8 @@ public class GoogleDriveController {
folderID = file.getId();
}
//check if folder already exist
public boolean checkFolder() {
// check if folder already exist
public boolean checkFolder() {
try {
Files.List request = service.files().list().setQ("mimeType = 'application/vnd.google-apps.folder' and name = 'cemu_savegames'");
FileList files = request.execute();
@ -215,52 +149,41 @@ public class GoogleDriveController {
}
}
//reading all local savegames
private void getLocalSavegames() throws IOException {
java.io.File dir = new java.io.File(saveDirectory);
String[] extensions = new String[] { "dat","sav","bin" };
localSavegames.removeAll(localSavegames);
localSavegamesName.removeAll(localSavegamesName);
LOGGER.info("Getting all dat,sav,bin files in " + dir.getCanonicalPath()+" including those in subdirectories");
List<java.io.File> files = (List<java.io.File>) FileUtils.listFiles(dir, extensions, true);
for (java.io.File file : files) {
localSavegamesName.add(file.getParentFile().getName()+"_"+file.getName());
localSavegames.add(file);
}
}
//reading all cloud savegames
private void getCloudSavegames() throws IOException {
LOGGER.info("getting all cloud savegames");
cloudSavegames.removeAll(cloudSavegames);
cloudSavegamesName.removeAll(cloudSavegamesName);
// FIXME it seams like there is a bug in this method
// get the name of the zip in the semu_savegames folder, which is the last upload Unix time
public long getLastCloudSync() throws IOException {
LOGGER.info("getting last cloud sync");
long lastCloudSync = 0;
Files.List request = service.files().list().setQ("'"+folderID+"' in parents").setFields("nextPageToken, files(id, name, size, modifiedTime, createdTime, md5Checksum)");
FileList files = request.execute();
for (File file : files.getFiles()) {
cloudSavegamesName.add(file.getName());
cloudSavegames.add(file);
}
downloadFile = file;
lastCloudSync = Long.parseLong(file.getName().substring(0, file.getName().length()-4));
}
return lastCloudSync;
}
private void getSavegamesFolderID() throws IOException {
Files.List request = service.files().list().setQ("mimeType = 'application/vnd.google-apps.folder' and name = 'cemu_savegames'");
FileList files = request.execute();
try {
LOGGER.info("FolderID: " + files.getFiles().get(0).getId());
setFolderID(files.getFiles().get(0).getId());
} catch (Exception e) {
LOGGER.error("Oops, something went wrong! It seems that you have more than one folder called 'cemu_savegames'!", e);
}
}
/**
* delete all existing files in cemu_savegames at first
* upload the new savegames zip file
* @param uploadFile savegames zip file
* @throws IOException
*/
public void uploadZipFile(java.io.File uploadFile) throws IOException{
LOGGER.info("deleting old savegames ...");
Files.List request = service.files().list().setQ("'"+folderID+"' in parents").setFields("nextPageToken, files(id, name, size, modifiedTime, createdTime, md5Checksum)");
FileList files = request.execute();
for (File file : files.getFiles()) {
service.files().delete(file.getId()).execute(); // deleting old file
}
//upload a file to the cloud from the local savegames folder
public void uploadFile(java.io.File uploadFile) throws IOException{
LOGGER.info("uploading " + uploadFile.getName() + " ...");
File fileMetadata = new File();
fileMetadata.setName(uploadFile.getParentFile().getName()+"_"+uploadFile.getName());
fileMetadata.setName(uploadFile.getName());
fileMetadata.setParents(Collections.singletonList(folderID));
fileMetadata.setModifiedTime(new DateTime(uploadFile.lastModified()));
FileContent mediaContent = new FileContent("", uploadFile);
@ -268,57 +191,16 @@ public class GoogleDriveController {
LOGGER.info("upload successfull, File ID: " + file.getId());
}
//download a file from the cloud to the local savegames folder
private void downloadFile(File downloadFile) throws IOException{
// download zip file from the cloud and unzip it
public java.io.File downloadZipFile(String cemu_UIDirectory) throws IOException{
LOGGER.info("downloading "+downloadFile.getName()+" ...");
java.io.File directory = new java.io.File(saveDirectory + "/" + downloadFile.getName().substring(0,8));
String file = downloadFile.getName().substring(9,downloadFile.getName().length());
if(!directory.exists()) {
LOGGER.info("dir dosent exist");
directory.mkdir();
}
java.io.File outputFile = new java.io.File(cemu_UIDirectory + "/" + downloadFile.getName());
OutputStream outputStream = new FileOutputStream(directory +"/"+ file);
OutputStream outputStream = new FileOutputStream(outputFile);
service.files().get(downloadFile.getId()).executeMediaAndDownloadTo(outputStream);
outputStream.close();
LOGGER.info("download successfull, File ID: " + file); //TODO add FileID
LOGGER.info("download successfull: " + downloadFile.getName());
return outputFile;
}
//update a file in the cloud, by deleting the old one and uploading an new with the same id
private void updateFile(File oldFile, java.io.File newFile) throws IOException {
LOGGER.info("updating " +oldFile.getName()+" ...");
service.files().delete(oldFile.getId()).execute(); //deleting old file
//uploading new file
File fileMetadata = new File();
fileMetadata.setName(newFile.getParentFile().getName()+"_"+newFile.getName());
fileMetadata.setParents(Collections.singletonList(folderID));
fileMetadata.setModifiedTime(new DateTime(newFile.lastModified()));
FileContent mediaContent = new FileContent("", newFile);
File file = service.files().create(fileMetadata, mediaContent).setFields("id, parents").execute();
LOGGER.info("update successfull, File ID: " + file.getId());
}
public void uploadAllFiles() {
try {
getLocalSavegames();
LOGGER.info("uploading " + localSavegames.size() + " files ...");
for (int i = 0; i < localSavegames.size(); i++) {
uploadFile(localSavegames.get(i));
}
LOGGER.info("finished uploading all files");
} catch (IOException e) {
LOGGER.error("error while uploading all files", e);
}
}
public String getFolderID() {
return folderID;
}
public void setFolderID(String folderID) {
this.folderID = folderID;
}
}