CHENXI LIU, University of British Columbia, Canada TOSHIKI AOKI, University of Tokyo, Japan MIKHAIL BESSMELTSEV, Université de Montréal, Canada ALLA SHEFFER, University of British Columbia, Canada
avoid the need for an unsustainably large manually annotated learning corpus by leveraging observations about artist workflow and perceptual cues viewers employ when mentally consolidating sketches.
Our method is the first to use a principled classification-based approach to vector sketch consolidation.
Limitation
Our performance is constrained by data scarcity which prevents greater reliance on context, due to overfitting concerns.
Dave Pagurek van Mossel, Chenxi Liu, Nicholas Vining, Mikhail Bessmeltsev, and Alla Sheffer. 2021. StrokeStrip: Joint Parameterization and Fitting of Stroke Clusters. ACM Trans. Graph. 40, 4 (July 2021), 50:1–50:18
Peng Xu, Timothy M Hospedales, Qiyue Yin, Yi-Zhe Song, Tao Xiang, and Liang Wang. 2022. Deep learning for free-hand sketch: A survey. IEEE Transactions on Pattern Analysis and Machine Intelligence (2022)
Jerry Yin, Chenxi Liu, Rebecca Lin, Nicholas Vining, Helge Rhodin, and Alla Sheffer. 2022. Detecting Viewer-Perceived Intended Vector Sketch Connectivity. ACM Transactions on Graphics 41 (2022). Issue 4.
Code
Since the author doesn’t upload codes, so I just guess the rough code framwork as follows:
//.h //used to get the likelyhood between different strips, so as to merge or split classRandomForestClassfier { //StrokeStrip parameterization[van Mossel et al. 2021]. public: RandomForestClassfier(); virtual ~RandomForestClassfier(); //stroke and stroke virtualfloatgetLikelihoodSS(Stroke& a, Stroke& b); //stroke and strip virtualfloatgetLikelihoodSP(Stroke& a, vector<Stroke>& b); //strip and strip virtualfloatgetLikelihoodPP(vector<Stroke>& a, vector<Stroke>& b); };
classLocalClassifier :public RandomForestClassfier { public: LocalClassifier(); ~LocalClassifier(); //stroke and stroke virtualfloatgetLikelihoodSS(Stroke& a, Stroke& b); //stroke and strip virtualfloatgetLikelihoodSP(Stroke& a, vector<Stroke>& b); //strip and strip virtualfloatgetLikelihoodPP(vector<Stroke>& a, vector<Stroke>& b); }; classGlobalClassifier :public RandomForestClassfier { public: //stroke and stroke virtualfloatgetLikelihoodSS(Stroke& a, Stroke& b); //stroke and strip virtualfloatgetLikelihoodSP(Stroke& a, vector<Stroke>& b); //strip and strip virtualfloatgetLikelihoodPP(vector<Stroke>& a, vector<Stroke>& b); }; classStroke { private: vector<Point> controlPoints; //each stroke is denoted by controlpoints, sorted by time //Time t; //the time this stroke is drawed by user vector<Point> samplePoints; int stripIndx; //which strip this stroke belongs to public: boolgetSamplePoints(float sampleRate); boolsetStripIndx(int idx); intgetStripIndx(); }; classVectorSketch { private: vector<Stroke> Strokes; //the vector Sketch consists of raw strokes vector<vector<Stroke>> Strips; //group the strokes to different strips int strokeWidth; public: boolSample(); boolLocalConsolidation(); boolGlobalConsolidation();
boolIsCompatible(vector<Stroke>& a, vector<Stroke>& b); boolMerge(vector<Stroke>& a, vector<Stroke>& b);
boolIsFormjectionByYin2022(vector<Stroke>& a, vector<Stroke>& b); voidFittingCurvesByVanMossel2021(); }; //main.cpp intmain() { //Input, get the information of strokes VectorSketch vectorSketch;
//Stroke.cpp boolStroke::getSamplePoints(float sampleRate) { for (float t = 0; t < 155; t += sampleRate) { Point p = Point(3 * t * t + 4 * (1 - t), 4 * t); //suppose the stroke function is like this samplePoints.emplace_back(p); } } boolStroke::setStripIndx(int idx) { stripIndx = idx; } intStroke::getStripIndx() { int stripIndx; } //VectorSketch.cpp boolVectorSketch::Sample() { float sampleRate = 1.2 * strokeWidth; for (autostroke : Strokes) { stroke.getSamplePoints(sampleRate); } //remove hook artifacts //.... } boolIsCompatible(vector<Stroke>& a, vector<Stroke>& b) { returnfalse; } //merge strip b to strip a boolMerge(vector<Stroke>& a, vector<Stroke>& b) { int stripIdxOfa = a[0].getStripIndx(); for (auto bb : b) { //update the information of strokes of strip b, and update the elements of strip a bb.setStripIndx(stripIdxOfa); a.emplace_back(bb); } returntrue; } boolIsFormjectionByYin2022(vector<Stroke>& a, vector<Stroke>& b) { //2022 Yin Algorithms //.. } voidFittingCurvesByVanMossel2021() { //2021 Van Mossel Algorithms //... } boolVectorSketch::LocalConsolidation() { auto Classfier = std::make_unique<LocalClassifier>(); int stripIdx = -1; for (int i = 0; i < Strokes.size(); i++) { Stroke curStroke = Strokes[i]; Stroke previousStroke = Strokes[i - 1]; vector<Stroke> previousStrip = Strips[previousStroke.getStripIndx()]; float probability = Classfier->getLikelihoodSP(curStroke, previousStrip);
//curStroke belongs to the sub-strip the previousStroke belongs to if (probability > 0.5) { //update curstroke information Strokes[i].setStripIndx(stripIdx); //update the previous strip previousStrip.emplace_back(curStroke); //proceed to the next stroke continue; } else { //save curStroke as a new sub-strip Strokes[i].setStripIndx(++stripIdx); Strips[stripIdx].emplace_back(curStroke);
//previousStrip has more than one stroke, decide if it can be merged with other strips. if (previousStrip.size() > 1) { int p_max = 0; int idx_max = -1; for (int j = 0; j < Strips.size(); j++) { //only call the classifier if two sub-strips are deemed compatible if (IsCompatible(Strips[j], previousStrip)) { float p = Classfier->getLikelihoodPP(Strips[j], previousStrip); if (p > p_max) { p_max = p; idx_max = j; } } } //merge the most similar two strips to one strip Merge(Strips[idx_max], previousStrip); Strips[stripIdx].clear(); --stripIdx; previousStrip.clear(); } } } //after the iteration above, the whole strokes are grouped into different strips, as the differnt-color curves in the figure.
//fitting curves to these strips FittingCurvesByVanMossel2021(); } boolVectorSketch::GlobalConsolidation() { auto Classfier = std::make_unique<GlobalClassifier>();
//strip reevaluation int n = Strips.size(); int StripIdx = n; for (int i = 0; i < n; i++) { vector<Stroke> strip = Strips[i]; //find the least likelihood two seed strokes in this strip int min_likelihood = INT_MAX;
vector<Stroke> seedStrokes(5); //Stroke seed_s1, seed_s2; for (int k = 0; k < strip.size(); k++) { for (int l = k + 1; l < strip.size(); l++) { Stroke s1 = strip[k]; Stroke s2 = strip[l]; float p = Classfier->getLikelihoodSS(s1, s2); if (p < min_likelihood) { min_likelihood = p; seedStrokes[0] = s1; seedStrokes[1] = s2; } } } //thie likelihood is high enougth, don't change this strip if (min_likelihood > 0.6) { continue; } else { //Reassign other strokes to sub-strip formed by seed_s1 or sub-strip formed by seed_s2 int newStripIdx = StripIdx; //seedStips.. vector<vector<Stroke>> seedStrips(5); seedStrokes[1].setStripIndx(newStripIdx); seedStrips[0].emplace_back(seedStrokes[0]); //seed_s1_strip seedStrips[1].emplace_back(seedStrokes[1]); //seed_s2_strip
for (autostroke : strip) { float p_max = 0; for (auto sub_strip : seedStrips) { float pi = Classfier->getLikelihoodSP(stroke, sub_strip); p_max = max(p_max, pi); } //this stroke doesn't belong to any seedStrip, so make this stroke as a new seed stroke if (p_max < 0.5) { seedStrokes.emplace_back(stroke); stroke.setStripIndx(++newStripIdx); } else { //assign this stroke to seed_s1_strip or seed_s2_strip or ..., according to the likelihood //... } } //merge this seedStrips according the likelihood //update Strips, cause may produce new strips in step above } }
//Connectivity Preservation classifier in Yin et al.[2022] for(auto strip : Strips) for (auto stripb : Strips) { if (IsFormjectionByYin2022(strip, strip)) { Merge(strip, strip); } }
//fitting curves to these strips FittingCurvesByVanMossel2021(); }
The classifier can be implemented as follow:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
//RandomForestClassfier.cpp floatRandomForestClassfier::getLikelihoodPP(vector<Stroke>& a, vector<Stroke>& b) { //call the python module, the trained model..
/* the classifier extract the features as follows: //Angles and Distance //Density //Narrows and Side-by-Side Extent //Evenness //1d Parameterization Distortion //Relative Precision */ return-1; }