Skip to content

Commit 188b2f5

Browse files
mbusch3mbusch3
committed
Adds an async batch fetch function for getting Canvas Pages.
1 parent acacb60 commit 188b2f5

File tree

2 files changed

+127
-18
lines changed

2 files changed

+127
-18
lines changed

‎src/Lms/Canvas/CanvasApi.php‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ class CanvasApi {
1111

1212
protected $session;
1313
protected $baseUrl;
14+
protected $apiToken;
1415
protected $httpClient;
1516

1617
public function __construct($baseUrl, $apiToken)
@@ -19,6 +20,7 @@ public function __construct($baseUrl, $apiToken)
1920
'headers' => ["Authorization: Bearer " . $apiToken],
2021
]);
2122
$this->baseUrl = $baseUrl;
23+
$this->apiToken = $apiToken;
2224
}
2325

2426
// API call GET
@@ -79,6 +81,64 @@ public function apiGet(string $url, array $options = [], int $perPage = 100, ?Lm
7981
return $lmsResponse;
8082
}
8183

84+
85+
// API call GET multiple (batch): Asynchronous requests
86+
public function apiGetBatch(array $paths): array
87+
{
88+
if(count($paths) == 0) {
89+
return [];
90+
}
91+
92+
$multi = curl_multi_init();
93+
$handles = [];
94+
$output = new ConsoleOutput();
95+
96+
// Create a handle for each path. These can be "watched" with curl_multi_exec
97+
foreach ($paths as $i => $url) {
98+
if (strpos($url, $this->baseUrl) === false) {
99+
$url = "https://{$this->baseUrl}/api/v1/{$url}";
100+
}
101+
$ch = curl_init($url);
102+
103+
curl_setopt_array($ch, [
104+
CURLOPT_RETURNTRANSFER => true,
105+
CURLOPT_HTTPHEADER => ["Authorization: Bearer {$this->apiToken}"],
106+
]);
107+
108+
curl_multi_add_handle($multi, $ch);
109+
$handles[$i] = $ch;
110+
}
111+
112+
// curl_multi_exec loop: See https://www.php.net/manual/en/function.curl-multi-exec.php#113002
113+
$running = null;
114+
do {
115+
curl_multi_exec($multi, $running);
116+
curl_multi_select($multi);
117+
} while ($running > 0);
118+
119+
// Gather results for each handle
120+
$responses = [];
121+
foreach ($handles as $i => $ch) {
122+
$content = curl_multi_getcontent($ch);
123+
$status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
124+
$error = curl_error($ch);
125+
126+
if($status == 200) {
127+
$responses[] = $content;
128+
}
129+
else {
130+
$output->writeln($status . " error fetching " . $paths[$i] . ": " . $error);
131+
}
132+
133+
curl_multi_remove_handle($multi, $ch);
134+
curl_close($ch);
135+
}
136+
137+
curl_multi_close($multi);
138+
return $responses;
139+
}
140+
141+
82142
public function apiPost($url, $options, $sendAuthorized = true)
83143
{
84144
$lmsResponse = new LmsResponse();

‎src/Lms/Canvas/CanvasLms.php‎

Lines changed: 67 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,11 @@ public function updateCourseContent(Course $course, User $user, $force = false):
180180
$apiToken = $this->getApiToken($user);
181181

182182
$canvasApi = new CanvasApi($apiDomain, $apiToken);
183+
$tempPages = [];
184+
$pageUrls = [];
185+
$asyncFetch = true;
183186

187+
$start_time = microtime(true);
184188
foreach ($urls as $contentType => $url) {
185189
$response = $canvasApi->apiGet($url);
186190

@@ -197,9 +201,13 @@ public function updateCourseContent(Course $course, User $user, $force = false):
197201
}
198202

199203
foreach ($contentList as $content) {
200-
if (('file' === $contentType) && (in_array($content['mime_class'], $this->util->getUnscannableFileMimeClasses()))) {
201-
$this->updateFileItem($course, $content);
202-
continue;
204+
205+
if ('file' === $contentType) {
206+
$output->writeln('Found ' . $content['mime_class'] . ' file: ' . $content['display_name']);
207+
if (in_array($content['mime_class'], $this->util->getUnscannableFileMimeClasses())) {
208+
$this->updateFileItem($course, $content);
209+
continue;
210+
}
203211
}
204212

205213
/* Quizzes should not be counted as assignments */
@@ -238,14 +246,31 @@ public function updateCourseContent(Course $course, User $user, $force = false):
238246
$output->writeln('New content item - ' . $contentType . ': ' . $lmsContent['title']);
239247
}
240248

241-
/* get page content */
249+
if (!$contentItem) {
250+
$contentItem = new ContentItem();
251+
$contentItem->setCourse($course)
252+
->setLmsContentId($lmsContent['id'])
253+
->setActive(true)
254+
->setContentType($contentType);
255+
$this->entityManager->persist($contentItem);
256+
}
257+
242258
if ('page' === $contentType) {
243-
$url = "courses/{$course->getLmsCourseId()}/pages/{$lmsContent['id']}";
244-
$pageResponse = $canvasApi->apiGet($url);
245-
$pageObj = $pageResponse->getContent();
259+
$url = "courses/{$course->getLmsCourseId()}/pages/{$lmsContent['id']}";
260+
if($asyncFetch) {
261+
/* NEW PAGE FETCH: New asynchronous batch fetch. The real magic is in the $pageUrls handler beneath this foreach loop (line ~305). */
262+
$tempContentItems[] = $contentItem;
263+
$pageUrls[] = $url;
264+
continue;
265+
}
266+
else {
267+
/* OLD PAGE FETCH: 1-at-a-time synchronous fetch */
268+
$pageResponse = $canvasApi->apiGet($url);
269+
$pageObj = $pageResponse->getContent();
246270

247-
if (!empty($pageObj['body'])) {
248-
$lmsContent['body'] = $pageObj['body'];
271+
if (!empty($pageObj['body'])) {
272+
$lmsContent['body'] = $pageObj['body'];
273+
}
249274
}
250275
}
251276

@@ -254,15 +279,6 @@ public function updateCourseContent(Course $course, User $user, $force = false):
254279
$lmsContent['body'] = file_get_contents($content['url']);
255280
}
256281

257-
if (!$contentItem) {
258-
$contentItem = new ContentItem();
259-
$contentItem->setCourse($course)
260-
->setLmsContentId($lmsContent['id'])
261-
->setActive(true)
262-
->setContentType($contentType);
263-
$this->entityManager->persist($contentItem);
264-
}
265-
266282
// some content types don't have an updated date, so we'll compare content
267283
// to find out if content has changed.
268284
if (in_array($contentType, ['syllabus', 'discussion_topic', 'announcement', 'quiz'])) {
@@ -281,9 +297,42 @@ public function updateCourseContent(Course $course, User $user, $force = false):
281297
}
282298
}
283299

300+
// If there are any pages to fetch, handle that now...
301+
if(count($pageUrls) > 0) {
302+
303+
$output->writeln('Fetching contents for ' . count($pageUrls) . ' pages asynchronously...');
304+
305+
// Request pages in a batch instead of synchronously
306+
$allPages = $canvasApi->apiGetBatch($pageUrls);
307+
308+
// Save indices for the tempContentItems array so it will be easier (O(1)) to match up...
309+
$tempContentItemsIndexById = [];
310+
foreach($tempContentItems as $index => $item) {
311+
$tempContentItemsIndexById[$item->getLmsContentId()] = $index;
312+
}
313+
314+
foreach($allPages as $pageData) {
315+
$lmsContent = $this->normalizeLmsContent($course, 'page', json_decode($pageData, true));
316+
317+
if (!empty($lmsContent['body'])) {
318+
$lmsContentId = $lmsContent['id'];
319+
// If the item exists in the tempContentItems array... Update and add to contentItems to scan.
320+
if(isset($tempContentItemsIndexById[$lmsContentId])) {
321+
$index = $tempContentItemsIndexById[$lmsContentId];
322+
$tempContentItems[$index]->update($lmsContent);
323+
$contentItems[] = $tempContentItems[$index];
324+
}
325+
}
326+
}
327+
}
328+
284329
// push any updates made to content items to DB
285330
$this->entityManager->flush();
286331

332+
// Log how long things took (compare synchronous vs asynchronous page fetch)
333+
$end_time = microtime(true);
334+
$output->writeln('updateCourseContent - time taken: ' . ($end_time - $start_time) . ' seconds');
335+
287336
return $contentItems;
288337
}
289338

0 commit comments

Comments
 (0)