Skip to content

Commit

Permalink
Merge pull request #1631 from elimu-ai/images-localhost
Browse files Browse the repository at this point in the history 
feat: Store images during localhost development
  • Loading branch information
@nya-elimu
nya-elimu authored Nov 3, 2023
2 parents bb7763d + ff86b86 commit 1e15fd7
Show file tree
Hide file tree
Showing 19 changed files with 3,770 additions and 3,653 deletions.
81 changes: 80 additions & 1 deletion src/main/java/ai/elimu/util/csv/CsvContentExtractionHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,19 @@
import ai.elimu.model.content.Number;
import ai.elimu.model.content.Sound;
import ai.elimu.model.content.Word;
import ai.elimu.model.content.multimedia.Image;
import ai.elimu.model.enums.ContentLicense;
import ai.elimu.model.v2.enums.ReadingLevel;
import ai.elimu.model.v2.enums.content.ImageFormat;
import ai.elimu.model.v2.enums.content.SpellingConsistency;
import ai.elimu.model.v2.enums.content.WordType;
import ai.elimu.model.v2.gson.content.ImageGson;
import ai.elimu.model.v2.gson.content.StoryBookChapterGson;
import ai.elimu.model.v2.gson.content.StoryBookGson;
import ai.elimu.model.v2.gson.content.StoryBookParagraphGson;
import ai.elimu.web.content.emoji.EmojiCsvExportController;
import ai.elimu.web.content.letter_sound.LetterSoundCsvExportController;
import ai.elimu.web.content.multimedia.image.ImageCsvExportController;
import ai.elimu.web.content.number.NumberCsvExportController;
import ai.elimu.web.content.storybook.StoryBookCsvExportController;
import ai.elimu.web.content.word.WordCsvExportController;
Expand All @@ -30,6 +34,9 @@
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.springframework.core.io.ClassRelativeResourceLoader;
import org.springframework.core.io.Resource;
import org.springframework.core.io.ResourceLoader;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -320,6 +327,68 @@ public static List<Emoji> getEmojisFromCsvBackup(File csvFile, WordDao wordDao)
return emojis;
}

/**
* For information on how the CSV files were generated, see {@link ImageCsvExportController#handleRequest}.
*/
public static List<Image> getImagesFromCsvBackup(File csvFile) {
logger.info("getImagesFromCsvBackup");

List<Image> images = new ArrayList<>();

Path csvFilePath = Paths.get(csvFile.toURI());
logger.info("csvFilePath: " + csvFilePath);
try {
Reader reader = Files.newBufferedReader(csvFilePath);
CSVFormat csvFormat = CSVFormat.DEFAULT
.withHeader(
"id",
"content_type",
"content_license",
"attribution_url",
"title",
"download_url",
"image_format"
)
.withSkipHeaderRecord();
CSVParser csvParser = new CSVParser(reader, csvFormat);
for (CSVRecord csvRecord : csvParser) {
logger.info("csvRecord: " + csvRecord);

Image image = new Image();

String contentType = csvRecord.get("content_type");
image.setContentType(contentType);

if (StringUtils.isNotBlank(csvRecord.get("content_license"))) {
ContentLicense contentLicense = ContentLicense.valueOf(csvRecord.get("content_license"));
image.setContentLicense(contentLicense);
}

String attributionUrl = csvRecord.get("attribution_url");
image.setAttributionUrl(attributionUrl);

String title = csvRecord.get("title");
image.setTitle(title);

ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvContentExtractionHelper.class);
Resource resource = resourceLoader.getResource("placeholder.png");
File bytesFile = resource.getFile();
Path bytesPath = bytesFile.toPath();
byte[] bytes = Files.readAllBytes(bytesPath);
image.setBytes(bytes);

ImageFormat imageFormat = ImageFormat.valueOf(csvRecord.get("image_format"));
image.setImageFormat(imageFormat);

images.add(image);
}
} catch (IOException ex) {
logger.error(ex);
}

return images;
}

/**
* For information on how the CSV files were generated, see {@link StoryBookCsvExportController#handleRequest}.
* <p />
Expand Down Expand Up @@ -375,7 +444,9 @@ public static List<StoryBookGson> getStoryBooksFromCsvBackup(File csvFile) {

if (StringUtils.isNotBlank(csvRecord.get("cover_image_id"))) {
Long coverImageId = Long.valueOf(csvRecord.get("cover_image_id"));
// storyBookGson.setCoverImage();
ImageGson coverImageGson = new ImageGson();
coverImageGson.setId(coverImageId);
storyBookGson.setCoverImage(coverImageGson);
}

List<StoryBookChapterGson> storyBookChapterGsons = new ArrayList<>();
Expand All @@ -388,6 +459,14 @@ public static List<StoryBookGson> getStoryBooksFromCsvBackup(File csvFile) {
StoryBookChapterGson storyBookChapterGson = new StoryBookChapterGson();
storyBookChapterGson.setSortOrder(chapterJsonObject.getInt("sortOrder"));

if (chapterJsonObject.has("image")) {
JSONObject chapterImageJsonObject = chapterJsonObject.getJSONObject("image");
logger.info("chapterImageJsonObject: " + chapterImageJsonObject);
ImageGson chapterImageGson = new ImageGson();
chapterImageGson.setId(chapterImageJsonObject.getLong("id"));
storyBookChapterGson.setImage(chapterImageGson);
}

List<StoryBookParagraphGson> storyBookParagraphGsons = new ArrayList<>();
JSONArray paragraphsJsonArray = chapterJsonObject.getJSONArray("storyBookParagraphs");
logger.info("paragraphsJsonArray: " + paragraphsJsonArray);
Expand Down
19 changes: 17 additions & 2 deletions src/main/java/ai/elimu/util/db/DbContentImportHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import ai.elimu.dao.ApplicationDao;
import ai.elimu.dao.ContributorDao;
import ai.elimu.dao.EmojiDao;
import ai.elimu.dao.ImageDao;
import ai.elimu.dao.LetterContributionEventDao;
import ai.elimu.dao.LetterDao;
import ai.elimu.dao.LetterSoundContributionEventDao;
Expand All @@ -27,6 +28,7 @@
import ai.elimu.model.content.StoryBookChapter;
import ai.elimu.model.content.StoryBookParagraph;
import ai.elimu.model.content.Word;
import ai.elimu.model.content.multimedia.Image;
import ai.elimu.model.contributor.Contributor;
import ai.elimu.model.contributor.LetterContributionEvent;
import ai.elimu.model.contributor.LetterSoundCorrespondenceContributionEvent;
Expand Down Expand Up @@ -81,6 +83,8 @@ public class DbContentImportHelper {

private EmojiDao emojiDao;

private ImageDao imageDao;

private StoryBookDao storyBookDao;

private StoryBookLearningEventDao storyBookLearningEventDao;
Expand Down Expand Up @@ -230,7 +234,13 @@ public synchronized void performDatabaseContentImport(Environment environment, L
}

// Extract and import Images from CSV file in src/main/resources/
// TODO
File imagesCsvFile = new File(contentDirectory, "images.csv");
List<Image> images = CsvContentExtractionHelper.getImagesFromCsvBackup(imagesCsvFile);
logger.info("images.size(): " + emojis.size());
imageDao = (ImageDao) webApplicationContext.getBean("imageDao");
for (Image image : images) {
imageDao.create(image);
}

// Extract and import Audios from CSV file in src/main/resources/
// TODO
Expand All @@ -251,14 +261,19 @@ public synchronized void performDatabaseContentImport(Environment environment, L
// TODO: storyBook.setContentLicense();
// TODO: storyBook.setAttributionUrl();
storyBook.setReadingLevel(storyBookGson.getReadingLevel());
Image coverImage = imageDao.read(storyBookGson.getCoverImage().getId());
storyBook.setCoverImage(coverImage);
storyBookDao.create(storyBook);

for (StoryBookChapterGson storyBookChapterGson : storyBookGson.getStoryBookChapters()) {
// Convert from GSON to JPA
StoryBookChapter storyBookChapter = new StoryBookChapter();
storyBookChapter.setStoryBook(storyBook);
storyBookChapter.setSortOrder(storyBookChapterGson.getSortOrder());
// TODO: storyBookChapter.setImage();
if (storyBookChapterGson.getImage() != null) {
Image chapterImage = imageDao.read(storyBookChapterGson.getImage().getId());
storyBookChapter.setImage(chapterImage);
}
storyBookChapterDao.create(storyBookChapter);

for (StoryBookParagraphGson storyBookParagraphGson : storyBookChapterGson.getStoryBookParagraphs()) {
Expand Down
45 changes: 34 additions & 11 deletions src/main/java/ai/elimu/web/content/multimedia/image/ImageCsvExportController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import ai.elimu.model.content.multimedia.Image;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
import java.util.List;

import org.apache.logging.log4j.Logger;
Expand All @@ -12,6 +13,9 @@
import org.springframework.web.bind.annotation.RequestMapping;

import javax.servlet.http.HttpServletResponse;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.logging.log4j.LogManager;
import org.springframework.web.bind.annotation.RequestMethod;

Expand All @@ -27,23 +31,42 @@ public class ImageCsvExportController {
@RequestMapping(value="/images.csv", method = RequestMethod.GET)
public void handleRequest(
HttpServletResponse response,
OutputStream outputStream) {
OutputStream outputStream) throws IOException {
logger.info("handleRequest");

// Generate CSV file
String csvFileContent = "id,content_type,content_license,attribution_url,title,download_url,image_format" + "\n";

List<Image> images = imageDao.readAll();
logger.info("images.size(): " + images.size());

CSVFormat csvFormat = CSVFormat.DEFAULT
.withHeader(
"id",
"content_type",
"content_license",
"attribution_url",
"title",
"download_url",
"image_format"
);
StringWriter stringWriter = new StringWriter();
CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);

for (Image image : images) {
String downloadUrl = "/image/" + image.getId() + "." + image.getImageFormat().toString().toLowerCase();
csvFileContent += image.getId() + ","
+ image.getContentType() + ","
+ image.getContentLicense()+ ","
+ "\"" + image.getAttributionUrl() + "\","
+ "\"" + image.getTitle() + "\","
+ "\"" + downloadUrl + "\","
+ image.getImageFormat() + "\n";

csvPrinter.printRecord(
image.getId(),
image.getContentType(),
image.getContentLicense(),
image.getAttributionUrl(),
image.getTitle(),
downloadUrl,
image.getImageFormat()
);

csvPrinter.flush();
}

String csvFileContent = stringWriter.toString();

response.setContentType("text/csv");
byte[] bytes = csvFileContent.getBytes();
Expand Down
Binary file added src/main/resources/ai/elimu/util/csv/placeholder.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit 1e15fd7

Please sign in to comment.